diff --git a/benchmarks/context-retrieval/corpora/corpus-express.json b/benchmarks/context-retrieval/corpora/corpus-express.json new file mode 100644 index 00000000..13c9d3eb --- /dev/null +++ b/benchmarks/context-retrieval/corpora/corpus-express.json @@ -0,0 +1,1121 @@ +{ + "schema_version": "codevetter.context-retrieval-corpus.v1", + "repository": { + "id": "express", + "head": "a3714473feb3d2908add734d340e7755fd85e0a3" + }, + "selection": { + "commits_scanned": 600, + "fixes_only": true, + "max_required_files": 8 + }, + "counts": { + "cases": 28, + "path_leak": 9, + "baseline_blind": 19, + "multi_file": 14 + }, + "cases": [ + { + "case_id": "express-05f40f4321fe", + "repository": "express", + "commit": "05f40f4321fee5235de0159dcd2a826c2532dff1", + "base_revision": "402e7f653fd79528df634c3464f2fe929e716ff3", + "authored_at": "2024-08-31T13:09:21-05:00", + "query": "content-disposition@^1.0.0 (#5884)", + "query_tokens": [ + "content", + "disposition" + ], + "specific_tokens": [ + "content", + "disposition" + ], + "required_files": [ + "package.json" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-18e5985b8a9d", + "repository": "express", + "commit": "18e5985b8a9d5e8423db0a9121f22bdaecd5b120", + "base_revision": "59e205a57a04fced6bb7b8ec0b5dec29461a9996", + "authored_at": "2026-06-16T06:06:31+03:00", + "query": "add Content-Length header only if Transfer-Encoding is not present (#4893)", + "query_tokens": [ + "content", + "encoding", + "header", + "length", + "present", + "transfer" + ], + "specific_tokens": [ + "content", + "encoding", + "header", + "length", + "present", + "transfer" + ], + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-246f6f5aeeba", + "repository": "express", + "commit": "246f6f5aeebad1b9d6a0ea08cd96d3fe19f8f23c", + "base_revision": "b11122be85377375465b2aa103da43301e56f128", + "authored_at": "2025-01-08T16:56:16+01:00", + "query": "Remove `utils-merge` dependency - use spread syntax instead (#6091)", + "query_tokens": [ + "dependency", + "instead", + "merge", + "remove", + "spread", + "syntax", + "utils" + ], + "specific_tokens": [ + "dependency", + "instead", + "merge", + "remove", + "spread", + "syntax", + "utils" + ], + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "derivable_files": [], + "incidental_files": [ + "package.json" + ], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-353348a83e68", + "repository": "express", + "commit": "353348a83e6874db7ded27102e91e7b6cad7e89c", + "base_revision": "dab6ee58229f2af3680023021703e7ab5cf9c76f", + "authored_at": "2020-02-20T07:34:41+01:00", + "query": "Fix typo in res.is JSDoc", + "query_tokens": [ + "jsdoc", + "typo" + ], + "specific_tokens": [ + "jsdoc", + "typo" + ], + "required_files": [ + "lib/request.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-3d10279826f5", + "repository": "express", + "commit": "3d10279826f59bf68e28995ce423f7bc4d2f11cf", + "base_revision": "5e9de5dcb6f1ff396038ded11951aa53c62a07a4", + "authored_at": "2018-09-19T23:25:16-04:00", + "query": "Fix issue where \"Request aborted\" may be logged in res.sendfile", + "query_tokens": [ + "aborted", + "issue", + "logged", + "request", + "sendfile" + ], + "specific_tokens": [ + "aborted", + "logged", + "request", + "sendfile" + ], + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-4c4f3ea10593", + "repository": "express", + "commit": "4c4f3ea1059319d217dbb8177dfec902d2917424", + "base_revision": "cb4c56e9a7ebc208730886e010b75475bf816fd9", + "authored_at": "2025-03-28T01:46:06+01:00", + "query": "serve-static@^2.2.0 (#6418)", + "query_tokens": [ + "serve", + "static" + ], + "specific_tokens": [ + "serve", + "static" + ], + "required_files": [ + "package.json" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-4e61d0100d90", + "repository": "express", + "commit": "4e61d0100d903de94673d4202eaa7edde5888fb4", + "base_revision": "7748475747c3ccabc929dac78ea4eeb34003b245", + "authored_at": "2024-08-31T11:06:25-05:00", + "query": "mime-types@^3.0.0 (#5882)", + "query_tokens": [ + "mime", + "types" + ], + "specific_tokens": [ + "mime", + "types" + ], + "required_files": [ + "test/res.type.js" + ], + "derivable_files": [], + "incidental_files": [ + "package.json" + ], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-54271f69b511", + "repository": "express", + "commit": "54271f69b511fea198471e6ff3400ab805d6b553", + "base_revision": "125bb742a38cd97938a3932b47cc301e41c31f5d", + "authored_at": "2024-09-09T17:16:58-05:00", + "query": "don't render redirect values in anchor href", + "query_tokens": [ + "anchor", + "href", + "redirect", + "render", + "values" + ], + "specific_tokens": [ + "anchor", + "href", + "redirect", + "render", + "values" + ], + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/res.redirect.js", + "tokens": [ + "redirect" + ] + } + ] + } + }, + { + "case_id": "express-5855339455a7", + "repository": "express", + "commit": "5855339455a7f60774bef4166829e742a5056fa8", + "base_revision": "1cc816993832eba829a2f556f7c08e27e6371301", + "authored_at": "2019-04-18T09:12:33-04:00", + "query": "Fix behavior of null/undefined as \"maxAge\" in res.cookie", + "query_tokens": [ + "behavior", + "cookie", + "null", + "undefined" + ], + "specific_tokens": [ + "behavior", + "cookie", + "null", + "undefined" + ], + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/res.cookie.js", + "tokens": [ + "cookie" + ] + } + ] + } + }, + { + "case_id": "express-62336717bfb6", + "repository": "express", + "commit": "62336717bfb6ff0ddca1c320cab76218928e7f93", + "base_revision": "3bbffdc41c1adebd1c4297e3661e2b182b8d27ac", + "authored_at": "2025-01-26T16:24:07+05:30", + "query": "added a missing semicolon in css styles in examples/auth (#6297)", + "query_tokens": [ + "added", + "auth", + "examples", + "missing", + "semicolon", + "styles" + ], + "specific_tokens": [ + "added", + "auth", + "examples", + "missing", + "semicolon", + "styles" + ], + "required_files": [ + "examples/auth/views/head.ejs" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "examples/auth/views/head.ejs", + "tokens": [ + "auth", + "examples" + ] + } + ] + } + }, + { + "case_id": "express-631ada0c645d", + "repository": "express", + "commit": "631ada0c645dc84c6df8788f5a7eb2b8100acea5", + "base_revision": "75e0c7a2c91665f44d053d83be15f8ecd0177f41", + "authored_at": "2022-04-29T13:34:47-04:00", + "query": "Fix hanging on large stack of sync routes", + "query_tokens": [ + "hanging", + "large", + "routes", + "stack", + "sync" + ], + "specific_tokens": [ + "hanging", + "large", + "routes", + "stack", + "sync" + ], + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-65b62065d2b4", + "repository": "express", + "commit": "65b62065d2b4308212fc3b19ab4986a9fa10ca41", + "base_revision": "0b243b1aee0d8774f2b97da2e8ac4340adaed1b3", + "authored_at": "2024-08-23T16:07:45-05:00", + "query": "fix(deps) serve-staic@2.0.0 (#5790)", + "query_tokens": [ + "deps", + "serve", + "staic" + ], + "specific_tokens": [ + "deps", + "serve", + "staic" + ], + "required_files": [ + "package.json" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-6cd404eb28ff", + "repository": "express", + "commit": "6cd404eb28ff861180f435b3015f8d0c8c0b44d4", + "base_revision": "3e81873b52e107898ed7ba45874959fb0546df3f", + "authored_at": "2026-01-07T15:41:34+01:00", + "query": "enhance req.acceptsCharsets method (#6088)", + "query_tokens": [ + "accepts", + "charsets", + "enhance", + "method" + ], + "specific_tokens": [ + "accepts", + "charsets", + "enhance", + "method" + ], + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/req.acceptsCharsets.js", + "tokens": [ + "accepts", + "charsets" + ] + } + ] + } + }, + { + "case_id": "express-6ed3439584b6", + "repository": "express", + "commit": "6ed3439584b6bc77b0f1156f8797700df063fa63", + "base_revision": "327af123a1833239adf7eb47fee94542b692d451", + "authored_at": "2025-02-14T16:51:27+01:00", + "query": "Update multiple links to use `https` instead of `http` (#6338)", + "query_tokens": [ + "http", + "https", + "instead", + "links", + "multiple", + "update" + ], + "specific_tokens": [ + "http", + "https", + "instead", + "links", + "multiple" + ], + "required_files": [ + ".editorconfig" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "Charter.md", + "Collaborator-Guide.md", + "Release-Process.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-708ac4cdf5cd", + "repository": "express", + "commit": "708ac4cdf5cd0a658d62490a9f4d78d3e1ec6612", + "base_revision": "92c5ce59f51cce4b3598fd040117772fac42dce8", + "authored_at": "2022-04-13T23:29:25-04:00", + "query": "Fix handling very large stacks of sync middleware", + "query_tokens": [ + "handling", + "large", + "middleware", + "stacks", + "sync", + "very" + ], + "specific_tokens": [ + "large", + "middleware", + "stacks", + "sync", + "very" + ], + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-74beeac0718c", + "repository": "express", + "commit": "74beeac0718c928b4ba249aba3652c52fbe32ca8", + "base_revision": "9bc1742937253825d2dc1e9a48c8e8424f0a315b", + "authored_at": "2023-02-23T17:23:22-05:00", + "query": "Fix routing requests without method", + "query_tokens": [ + "method", + "requests", + "routing" + ], + "specific_tokens": [ + "method", + "requests", + "routing" + ], + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-7ec5dd2b3c5e", + "repository": "express", + "commit": "7ec5dd2b3c5e7379f68086dae72859f5573c8b9b", + "base_revision": "ab2c70b954ac2ceb3aaf466b0f59089999952dd0", + "authored_at": "2022-05-20T09:37:20-04:00", + "query": "Fix regression routing a large stack in a single route", + "query_tokens": [ + "large", + "regression", + "route", + "routing", + "single", + "stack" + ], + "specific_tokens": [ + "large", + "regression", + "route", + "routing", + "single", + "stack" + ], + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "lib/router/route.js", + "tokens": [ + "route" + ] + }, + { + "path": "test/Route.js", + "tokens": [ + "route" + ] + } + ] + } + }, + { + "case_id": "express-925a1dff1e42", + "repository": "express", + "commit": "925a1dff1e42f1b393c977b8b77757fcf633e09f", + "base_revision": "9c85a25c02e83ad16e1561d02c8ede652f0ef15b", + "authored_at": "2026-02-21T22:11:08-05:00", + "query": "bump qs minimum to ^6.14.2 for CVE-2026-2391 (#7057)", + "query_tokens": [ + "bump", + "minimum" + ], + "specific_tokens": [ + "bump", + "minimum" + ], + "required_files": [ + "package.json" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-99a369f3d51b", + "repository": "express", + "commit": "99a369f3d51bafcf0c09657250067249f19d04f5", + "base_revision": "a1dbb113779c45a10f56a910de1bc5f7e750eaea", + "authored_at": "2020-03-05T12:00:08+01:00", + "query": "Fix incorrect middleware execution with unanchored RegExps", + "query_tokens": [ + "execution", + "exps", + "incorrect", + "middleware", + "unanchored" + ], + "specific_tokens": [ + "execution", + "exps", + "incorrect", + "middleware", + "unanchored" + ], + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-9d8223d92ee8", + "repository": "express", + "commit": "9d8223d92ee81137a50a28eb6ad55a096791091d", + "base_revision": "90ec6206d3275fdf2787f2028c6f48a2d92e8042", + "authored_at": "2026-06-16T08:04:06+05:30", + "query": "replace deprecated trimRight() with trimEnd() (#7265)", + "query_tokens": [ + "deprecated", + "replace", + "right", + "trim" + ], + "specific_tokens": [ + "deprecated", + "replace", + "right", + "trim" + ], + "required_files": [ + "lib/request.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-9dd0e7afdb6d", + "repository": "express", + "commit": "9dd0e7afdb6d022e18add1e009c4e3a66258c1fa", + "base_revision": "1b2f3a06982cf4d51482dd79447b66315687b287", + "authored_at": "2021-11-16T23:58:17-05:00", + "query": "Fix handling of undefined in res.jsonp", + "query_tokens": [ + "handling", + "jsonp", + "undefined" + ], + "specific_tokens": [ + "jsonp", + "undefined" + ], + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/res.jsonp.js", + "tokens": [ + "jsonp" + ] + } + ] + } + }, + { + "case_id": "express-a1dbb113779c", + "repository": "express", + "commit": "a1dbb113779c45a10f56a910de1bc5f7e750eaea", + "base_revision": "353348a83e6874db7ded27102e91e7b6cad7e89c", + "authored_at": "2020-01-08T10:56:45+09:00", + "query": "Fix res.jsonp(obj, status) deprecation message", + "query_tokens": [ + "deprecation", + "jsonp", + "message", + "status" + ], + "specific_tokens": [ + "deprecation", + "jsonp", + "message", + "status" + ], + "required_files": [ + "lib/response.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-b3906cbdded2", + "repository": "express", + "commit": "b3906cbdded224554ce1c778996d77b5d2a15ebc", + "base_revision": "fed8c2a8857a30b91e07a37f739aee18e756853f", + "authored_at": "2024-09-09T23:32:19-05:00", + "query": "serve-static@^2.1.0", + "query_tokens": [ + "serve", + "static" + ], + "specific_tokens": [ + "serve", + "static" + ], + "required_files": [ + "package.json" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-d12772393c82", + "repository": "express", + "commit": "d12772393c8249666e0fcc11e298f48ce4632a11", + "base_revision": "6b7ccfcf120e77f20aa22ff96ccf98a994c7d35b", + "authored_at": "2026-02-01T03:12:23Z", + "query": "search example to support Redis v4+ and Express 4/5 (#6274)", + "query_tokens": [ + "example", + "express", + "redis", + "search", + "support" + ], + "specific_tokens": [ + "example", + "redis", + "search", + "support" + ], + "required_files": [ + "examples/search/index.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "examples/search/index.js", + "tokens": [ + "search" + ] + } + ] + } + }, + { + "case_id": "express-f275e87dff1a", + "repository": "express", + "commit": "f275e87dff1aaef86080e6931888de4968585fd8", + "base_revision": "9dd0e7afdb6d022e18add1e009c4e3a66258c1fa", + "authored_at": "2021-11-03T10:58:25Z", + "query": "Fix handling of undefined when \"json escape\" is enabled", + "query_tokens": [ + "enabled", + "escape", + "handling", + "json", + "undefined" + ], + "specific_tokens": [ + "enabled", + "escape", + "json", + "undefined" + ], + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "test/res.json.js", + "tokens": [ + "json" + ] + } + ] + } + }, + { + "case_id": "express-f3fa758af966", + "repository": "express", + "commit": "f3fa758af9664526ee18c58764590e48f559e6ae", + "base_revision": "ede24da9645ed771ddeb9cf327d042d35a31d42d", + "authored_at": "2018-05-26T21:58:15+02:00", + "query": "Fix JSDoc for Router constructor", + "query_tokens": [ + "constructor", + "jsdoc", + "router" + ], + "specific_tokens": [ + "constructor", + "jsdoc", + "router" + ], + "required_files": [ + "lib/router/index.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "lib/router/index.js", + "tokens": [ + "router" + ] + } + ] + } + }, + { + "case_id": "express-f9954dd317a1", + "repository": "express", + "commit": "f9954dd317a1b534e62a5ae3aa7f52f3582b8881", + "base_revision": "5da5a11a498a3034623960861e542a3b39b00c94", + "authored_at": "2025-04-17T00:18:38+08:00", + "query": "remove duplicate word (#6456)", + "query_tokens": [ + "duplicate", + "remove", + "word" + ], + "specific_tokens": [ + "duplicate", + "remove", + "word" + ], + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "express-fed8c2a8857a", + "repository": "express", + "commit": "fed8c2a8857a30b91e07a37f739aee18e756853f", + "base_revision": "bdd81f8670975ef30fd49e92513ef48d35029eaf", + "authored_at": "2024-09-09T22:32:07-05:00", + "query": "body-parser@^2.0.1", + "query_tokens": [ + "body", + "parser" + ], + "specific_tokens": [ + "body", + "parser" + ], + "required_files": [ + "package.json" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "History.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + } + ], + "rejected_sample": [ + { + "commit": "a3714473feb3d2908add734d340e7755fd85e0a3", + "subject": "build(deps-dev): bump hbs from 4.2.0 to 4.2.1 (#7152)", + "reason": "not-a-fix" + }, + { + "commit": "ae6dd37680e3a00618d6c8a3e522f0ee4eeba1a4", + "subject": "feat: allow conditional revalidation for QUERY requests (#7366)", + "reason": "not-a-fix" + }, + { + "commit": "ba006766fb964571723138708eacaba0f55759cd", + "subject": "build(deps-dev): bump morgan from 1.10.1 to 1.11.0 (#7353)", + "reason": "not-a-fix" + }, + { + "commit": "5175d2f357e9c6fac998812b3fc22a1a90ead988", + "subject": "build(deps): bump actions/checkout from 6.0.2 to 7.0.0 (#7345)", + "reason": "not-a-fix" + }, + { + "commit": "66878d3e70437ba7b887ec519a3e33edc5bca0c7", + "subject": "docs: use the new logo (#7316)", + "reason": "excluded-subject" + }, + { + "commit": "59e205a57a04fced6bb7b8ec0b5dec29461a9996", + "subject": "Upgrade `content-disposition` (#7233)", + "reason": "not-a-fix" + }, + { + "commit": "b3004cb8c825321279efa48d6ad115d91a6c1b83", + "subject": "build(deps): bump github/codeql-action from 4.35.2 to 4.36.0 (#7297)", + "reason": "not-a-fix" + }, + { + "commit": "90ec6206d3275fdf2787f2028c6f48a2d92e8042", + "subject": "Improve error logging by logging full error object (#6464)", + "reason": "not-a-fix" + }, + { + "commit": "cb19f04170fc815321d40dcd3bc620ae03c828cd", + "subject": "Upgrade `content-type` (#7234)", + "reason": "not-a-fix" + }, + { + "commit": "a08da78e64a7fdc610d453fb7771754091d8ed3b", + "subject": "deps: bump qs minimum to 6.15.2 (#7305)", + "reason": "excluded-subject" + }, + { + "commit": "dae209ae6559c29cfca2a1f4414c51d89ea643d5", + "subject": "build(deps): bump github/codeql-action from 4.35.1 to 4.35.2 (#7212)", + "reason": "not-a-fix" + }, + { + "commit": "777001a0f52aaf761f8b95afc02bee19c87a0158", + "subject": "build(deps): bump actions/upload-artifact from 7.0.0 to 7.0.1 (#7211)", + "reason": "not-a-fix" + }, + { + "commit": "64576bde91c6fbe8214006207f56cb84e7ab9274", + "subject": "build(deps): bump actions/setup-node from 6.3.0 to 6.4.0 (#7210)", + "reason": "not-a-fix" + }, + { + "commit": "f5c159b112e8ac6e3b2b022eff03ad727d4ff98a", + "subject": "ci: build express with node.js v26 (#7218)", + "reason": "not-a-fix" + }, + { + "commit": "2eae22b1e12df345e314a4db483052a780e0ba0b", + "subject": "chore: ensure safe config in the npmrc (#7144)", + "reason": "excluded-subject" + }, + { + "commit": "f873ac23124ffcff8c040b4bd257b32c29828d53", + "subject": "fixed typo in history.md (#7191)", + "reason": "no-code-files" + }, + { + "commit": "6340c1eaaedc0ddcae8be8df2cdb1d2e961cbf2f", + "subject": "build(deps): bump github/codeql-action from 4.32.4 to 4.35.1 (#7150)", + "reason": "not-a-fix" + }, + { + "commit": "8cc3afa8e35e1a62ccf48276d456278455eb784d", + "subject": "build(deps): bump actions/setup-node from 6.2.0 to 6.3.0 (#7149)", + "reason": "not-a-fix" + }, + { + "commit": "e7fd63a3878596154dd0693e92a8e5e41a45647c", + "subject": "build(deps): bump actions/download-artifact from 8.0.0 to 8.0.1 (#7148)", + "reason": "not-a-fix" + }, + { + "commit": "8e022edc9185f540a3fcecaf5e56b850d919cdac", + "subject": "docs: update npm install docs URL in Readme.md (#7159)", + "reason": "excluded-subject" + } + ] +} diff --git a/benchmarks/context-retrieval/corpora/corpus-fastify.json b/benchmarks/context-retrieval/corpora/corpus-fastify.json new file mode 100644 index 00000000..ccb670b4 --- /dev/null +++ b/benchmarks/context-retrieval/corpora/corpus-fastify.json @@ -0,0 +1,3546 @@ +{ + "schema_version": "codevetter.context-retrieval-corpus.v1", + "repository": { + "id": "fastify", + "head": "83e69762aff047940b376fc2a266d143499ef727" + }, + "selection": { + "commits_scanned": 600, + "fixes_only": true, + "max_required_files": 8 + }, + "counts": { + "cases": 82, + "path_leak": 47, + "baseline_blind": 35, + "multi_file": 66 + }, + "cases": [ + { + "case_id": "fastify-01ca8393b9d1", + "repository": "fastify", + "commit": "01ca8393b9d10a897f0d2daedbe2f9eb90076ef5", + "base_revision": "b6cdc6a40598d15f38fe7d399eb17f4b84b7bf06", + "authored_at": "2026-07-30T17:20:20+08:00", + "query": "uncatchable throws in writeHead when using async hook (#6881)", + "query_tokens": [ + "async", + "head", + "hook", + "throws", + "uncatchable", + "using", + "write" + ], + "specific_tokens": [ + "async", + "head", + "hook", + "throws", + "uncatchable", + "using", + "write" + ], + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-0dfa46c46687", + "repository": "fastify", + "commit": "0dfa46c466875ca1e2338c5ed0baf1359174fbcb", + "base_revision": "6cbcfa7d82aaf25c7e53804e2a024c531d15c4d0", + "authored_at": "2025-03-24T19:07:23+01:00", + "query": "double hook execution (#6013)", + "query_tokens": [ + "double", + "execution", + "hook" + ], + "specific_tokens": [ + "double", + "execution", + "hook" + ], + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-0e9a3b35d8b1", + "repository": "fastify", + "commit": "0e9a3b35d8b13c9f261bf29bef46168a5d43c28f", + "base_revision": "b93d611c7cefc2cd4a0b6f05cc864f2b6c397a6d", + "authored_at": "2026-07-30T12:00:13+02:00", + "query": "add http method override warning (#6879)", + "query_tokens": [ + "http", + "method", + "override", + "warning" + ], + "specific_tokens": [ + "http", + "method", + "override", + "warning" + ], + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "docs/Reference/Server.md", + "docs/Reference/Warnings.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.2, + "shared_tokens": [ + { + "path": "test/http-methods/custom-http-methods.test.js", + "tokens": [ + "http" + ] + } + ] + } + }, + { + "case_id": "fastify-0f54a40f63b3", + "repository": "fastify", + "commit": "0f54a40f63b399bfe06a3deeab832f8b5feb6345", + "base_revision": "0e9a3b35d8b13c9f261bf29bef46168a5d43c28f", + "authored_at": "2026-08-02T13:01:44+02:00", + "query": "allow explicit http2: false in server options (#6888)", + "query_tokens": [ + "allow", + "explicit", + "false", + "http2", + "options", + "server" + ], + "specific_tokens": [ + "allow", + "explicit", + "false", + "http2", + "options", + "server" + ], + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-120661fab3b2", + "repository": "fastify", + "commit": "120661fab3b2de662444a835cb3f58a1c4851b8c", + "base_revision": "de610ccc94652719c5c8477344c6ae7dfe127c11", + "authored_at": "2025-01-30T19:14:18+01:00", + "query": "missing supportedMethods (#5970)", + "query_tokens": [ + "methods", + "missing", + "supported" + ], + "specific_tokens": [ + "methods", + "missing", + "supported" + ], + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-153d78aaf952", + "repository": "fastify", + "commit": "153d78aaf95249e275e234185d3e2d1c770859c4", + "base_revision": "57aad569180eb67458562691ab51bbe3d7bd62c5", + "authored_at": "2026-01-16T09:18:33+01:00", + "query": "updated version in the fastify.js (#6446)", + "query_tokens": [ + "fastify", + "updated", + "version" + ], + "specific_tokens": [ + "updated", + "version" + ], + "required_files": [ + "fastify.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "fastify.js", + "tokens": [ + "fastify" + ] + } + ] + } + }, + { + "case_id": "fastify-15c2fb193346", + "repository": "fastify", + "commit": "15c2fb1933461f846a6160923a2bc4e091039826", + "base_revision": "eaff726a68c629c1aedc99f96b3c4b95dfdb38d1", + "authored_at": "2026-08-13T17:32:28+02:00", + "query": "return undefined for invalid media types (#6942)", + "query_tokens": [ + "invalid", + "media", + "return", + "types", + "undefined" + ], + "specific_tokens": [ + "invalid", + "media", + "return", + "types", + "undefined" + ], + "required_files": [ + "lib/content-type.js" + ], + "derivable_files": [ + "test/content-type.test.js" + ], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-16a74e7cb2d1", + "repository": "fastify", + "commit": "16a74e7cb2d1abdc2feca9097f2936fe8b851a5c", + "base_revision": "27938ac777c15482a81881b29f86c82b025f7fae", + "authored_at": "2026-08-07T23:37:50-07:00", + "query": "reset lastIndex before testing global/sticky content-type RegExp parsers (#6846)", + "query_tokens": [ + "content", + "global", + "index", + "last", + "parsers", + "reset", + "sticky", + "testing", + "type" + ], + "specific_tokens": [ + "content", + "global", + "index", + "last", + "parsers", + "reset", + "sticky", + "testing", + "type" + ], + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "lib/content-type-parser.js", + "tokens": [ + "content", + "type" + ] + }, + { + "path": "test/content-parser.test.js", + "tokens": [ + "content" + ] + } + ] + } + }, + { + "case_id": "fastify-16b2e4d01888", + "repository": "fastify", + "commit": "16b2e4d018885a732aa0702bf708f4f790f5ef7f", + "base_revision": "0c7abb1eb55d8cae7c73789d8c400a5d1603d37b", + "authored_at": "2025-05-17T18:50:04+07:00", + "query": "add missing version to request.routeOptions (#6126)", + "query_tokens": [ + "missing", + "options", + "request", + "route", + "version" + ], + "specific_tokens": [ + "missing", + "options", + "request", + "route", + "version" + ], + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/types/request.test-d.ts", + "tokens": [ + "request" + ] + }, + { + "path": "types/request.d.ts", + "tokens": [ + "request" + ] + } + ] + } + }, + { + "case_id": "fastify-16c2c3db5cf0", + "repository": "fastify", + "commit": "16c2c3db5cf0fedba34860ae68399ae7f82a0787", + "base_revision": "eb428803e9c0b3b482fcfd094fa55c7c81ff9c30", + "authored_at": "2025-06-10T16:55:01+02:00", + "query": "do not freeze request.routeOptions (#6141)", + "query_tokens": [ + "freeze", + "options", + "request", + "route" + ], + "specific_tokens": [ + "freeze", + "options", + "request", + "route" + ], + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "lib/request.js", + "tokens": [ + "request" + ] + }, + { + "path": "test/request-error.test.js", + "tokens": [ + "request" + ] + } + ] + } + }, + { + "case_id": "fastify-1d5f8ae199ed", + "repository": "fastify", + "commit": "1d5f8ae199edddea68f6df4c5d68b62279e6c59d", + "base_revision": "c02c5f5696c8695184f355891106ec26139cb559", + "authored_at": "2026-01-03T14:34:18+01:00", + "query": "ajv options type validation (#6437)", + "query_tokens": [ + "options", + "type", + "validation" + ], + "specific_tokens": [ + "options", + "type", + "validation" + ], + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "derivable_files": [], + "incidental_files": [ + "package.json" + ], + "excluded_files": [ + "docs/Reference/Server.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-1e0be1294e85", + "repository": "fastify", + "commit": "1e0be1294e85734f1e6ac651f3ff664d389d7799", + "base_revision": "8fad07272bf8042db8c0074c6fd40ed50fb6b0f7", + "authored_at": "2026-02-28T09:32:19Z", + "query": "remove format placeholder from FST_ERR_CTP_INVALID_MEDIA_TYPE message (#6528)", + "query_tokens": [ + "format", + "invalid", + "media", + "message", + "placeholder", + "remove", + "type" + ], + "specific_tokens": [ + "format", + "invalid", + "media", + "message", + "placeholder", + "remove", + "type" + ], + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "derivable_files": [ + "test/internals/errors.test.js" + ], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "lib/content-type-parser.js", + "tokens": [ + "type" + ] + } + ] + } + }, + { + "case_id": "fastify-21b4c3c10103", + "repository": "fastify", + "commit": "21b4c3c101038030e552ad2cfcca3f43a349f464", + "base_revision": "9bbf60978d013c26663b7be8031ecc12e6db12bd", + "authored_at": "2026-04-24T08:59:22+02:00", + "query": "guard forwarded host and protocol on socket presence (#6684)", + "query_tokens": [ + "forwarded", + "guard", + "host", + "presence", + "protocol", + "socket" + ], + "specific_tokens": [ + "forwarded", + "guard", + "host", + "presence", + "protocol", + "socket" + ], + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "derivable_files": [ + "test/internals/request.test.js" + ], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-27938ac777c1", + "repository": "fastify", + "commit": "27938ac777c15482a81881b29f86c82b025f7fae", + "base_revision": "a2f590563f111ebc8e92b8f17701909e9b19d4b7", + "authored_at": "2026-08-08T09:36:43+03:00", + "query": "recognize constructor-assigned built-in properties as decorator dependencies (#6892)", + "query_tokens": [ + "assigned", + "built", + "constructor", + "decorator", + "dependencies", + "properties", + "recognize" + ], + "specific_tokens": [ + "assigned", + "built", + "constructor", + "decorator", + "dependencies", + "properties", + "recognize" + ], + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/decorator-instance-properties.test.js", + "tokens": [ + "decorator", + "properties" + ] + } + ] + } + }, + { + "case_id": "fastify-29bdcbb91f13", + "repository": "fastify", + "commit": "29bdcbb91f1318dc90ae8c0e52ab6f4e13c42744", + "base_revision": "01ca8393b9d10a897f0d2daedbe2f9eb90076ef5", + "authored_at": "2026-07-30T12:22:03+03:00", + "query": "honor quoted-string in Content-Type parameter values (#6865)", + "query_tokens": [ + "content", + "honor", + "parameter", + "quoted", + "string", + "type", + "values" + ], + "specific_tokens": [ + "content", + "honor", + "parameter", + "quoted", + "string", + "type", + "values" + ], + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "derivable_files": [ + "test/content-type.test.js" + ], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "lib/content-type.js", + "tokens": [ + "content", + "type" + ] + } + ] + } + }, + { + "case_id": "fastify-2ae3e1d73491", + "repository": "fastify", + "commit": "2ae3e1d73491c023080efed566204c470cb49fd0", + "base_revision": "d76dbcd58bf6d82bba167cdc74a79d2f74dbcb97", + "authored_at": "2026-04-21T05:38:51-04:00", + "query": "validate invalid route logLevel at registration (#6523)", + "query_tokens": [ + "invalid", + "level", + "registration", + "route", + "validate" + ], + "specific_tokens": [ + "invalid", + "level", + "registration", + "route", + "validate" + ], + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "derivable_files": [ + "test/internals/errors.test.js" + ], + "incidental_files": [], + "excluded_files": [ + "docs/Reference/Errors.md", + "docs/Reference/Routes.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.25, + "shared_tokens": [ + { + "path": "lib/route.js", + "tokens": [ + "route" + ] + } + ] + } + }, + { + "case_id": "fastify-2af83d64b53a", + "repository": "fastify", + "commit": "2af83d64b53a0a786141d93edb9ffefac8b6446a", + "base_revision": "5c14e05670c80455b99286ee0b6eac05eabec831", + "authored_at": "2026-01-24T13:11:54Z", + "query": "Fix MIT Licence file to conform to standard (#6464)", + "query_tokens": [ + "conform", + "file", + "licence", + "standard" + ], + "specific_tokens": [ + "conform", + "file", + "licence", + "standard" + ], + "required_files": [ + "LICENSE" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-2bf4265b991a", + "repository": "fastify", + "commit": "2bf4265b991aa4c9a2d3886882be6f062338c7bc", + "base_revision": "fc9eafddae37dea167d2affc26a29a04f9412047", + "authored_at": "2026-06-07T15:19:47+02:00", + "query": "chunk large HTTP/2 buffer replies (#6746)", + "query_tokens": [ + "buffer", + "chunk", + "http", + "large", + "replies" + ], + "specific_tokens": [ + "buffer", + "chunk", + "http", + "large", + "replies" + ], + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "docs/Reference/Reply.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-2c0be02b396b", + "repository": "fastify", + "commit": "2c0be02b396bdd37bc34ee49356003be3c5a3f95", + "base_revision": "1b6ddaaa888833f7192fe721e9ecc292e410d344", + "authored_at": "2025-03-27T23:30:08+07:00", + "query": "wrong reply return type (#6026)", + "query_tokens": [ + "reply", + "return", + "type", + "wrong" + ], + "specific_tokens": [ + "reply", + "return", + "type" + ], + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/types/type-provider.test-d.ts", + "tokens": [ + "type" + ] + }, + { + "path": "types/type-provider.d.ts", + "tokens": [ + "type" + ] + } + ] + } + }, + { + "case_id": "fastify-2f597a9297a3", + "repository": "fastify", + "commit": "2f597a9297a3020a9f30bf5548a8442c28ddb1c1", + "base_revision": "8b9c07b645a8156c23a1d2619267fc84ea879250", + "authored_at": "2026-06-21T07:54:17-05:00", + "query": "avoid double slash when joining nested prefixes (#6803)", + "query_tokens": [ + "avoid", + "double", + "joining", + "nested", + "prefixes", + "slash" + ], + "specific_tokens": [ + "avoid", + "double", + "joining", + "nested", + "prefixes", + "slash" + ], + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-325682c9e5ab", + "repository": "fastify", + "commit": "325682c9e5abb4dce494770c7cdf27139934347d", + "base_revision": "ff7eff5eec8a820b2c6f79b477e1a784bbf42341", + "authored_at": "2026-07-26T20:59:25+08:00", + "query": "correctly update falsy values from validator (#6483)", + "query_tokens": [ + "correctly", + "falsy", + "update", + "validator", + "values" + ], + "specific_tokens": [ + "falsy", + "validator", + "values" + ], + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-329e40f09f23", + "repository": "fastify", + "commit": "329e40f09f23910ed0193a9636b474844a7fbe79", + "base_revision": "34fea68924d1b8bec34b18b5317c71f9006790f1", + "authored_at": "2025-09-26T16:56:39-03:00", + "query": "error throwing in reply (#6299)", + "query_tokens": [ + "error", + "reply", + "throwing" + ], + "specific_tokens": [ + "reply", + "throwing" + ], + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "lib/reply.js", + "tokens": [ + "reply" + ] + }, + { + "path": "test/reply-web-stream-locked.test.js", + "tokens": [ + "reply" + ] + } + ] + } + }, + { + "case_id": "fastify-36498f85a91c", + "repository": "fastify", + "commit": "36498f85a91cebc107282fe8128e08e48567bde0", + "base_revision": "4e5a3631ec731fa9b82689e140b399f21a8b96fd", + "authored_at": "2025-09-22T17:19:05+05:00", + "query": "close http2 sessions on close server (#6137)", + "query_tokens": [ + "close", + "http2", + "server", + "sessions" + ], + "specific_tokens": [ + "close", + "http2", + "server", + "sessions" + ], + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "docs/Reference/Server.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "lib/server.js", + "tokens": [ + "server" + ] + }, + { + "path": "test/http2/closing.test.js", + "tokens": [ + "http2" + ] + } + ] + } + }, + { + "case_id": "fastify-37c7ca89d88f", + "repository": "fastify", + "commit": "37c7ca89d88f55880e6fe23c6d5d8b26c048538f", + "base_revision": "787b7a7c2e00f2fa211a3e52d134ca1e8ce81530", + "authored_at": "2025-01-09T09:42:41+01:00", + "query": "don't check for payload type in default json parser (#5933)", + "query_tokens": [ + "check", + "default", + "json", + "parser", + "payload", + "type" + ], + "specific_tokens": [ + "check", + "default", + "json", + "parser", + "payload", + "type" + ], + "required_files": [ + "lib/contentTypeParser.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "lib/contentTypeParser.js", + "tokens": [ + "parser", + "type" + ] + } + ] + } + }, + { + "case_id": "fastify-39f0f24233cf", + "repository": "fastify", + "commit": "39f0f24233cf6da2fef48551f51be2f589f7d5d0", + "base_revision": "af92d0d2a9280a41e23fea4dc2b5cfff41169d5d", + "authored_at": "2026-03-23T07:25:33-04:00", + "query": "Fix port parsing (#6603)", + "query_tokens": [ + "parsing", + "port" + ], + "specific_tokens": [ + "parsing", + "port" + ], + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/request-port.test.js", + "tokens": [ + "port" + ] + } + ] + } + }, + { + "case_id": "fastify-39f35f9a0596", + "repository": "fastify", + "commit": "39f35f9a0596537326b960183b1aa64aca796632", + "base_revision": "70b14e92c0b55e8201f5530ba2e6bab4e928c784", + "authored_at": "2025-09-09T23:34:30+02:00", + "query": "fix typo of deprecation warning FSTDEP022 (#6313)", + "query_tokens": [ + "deprecation", + "fstdep022", + "typo", + "warning" + ], + "specific_tokens": [ + "deprecation", + "fstdep022", + "typo", + "warning" + ], + "required_files": [ + "lib/warnings.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-3b0f76993d51", + "repository": "fastify", + "commit": "3b0f76993d51f8db662814c693f4ebea8d97cc95", + "base_revision": "e4474cfd794b1ff3cf4cbdb60c465c69fed4db1d", + "authored_at": "2026-03-07T09:44:16+01:00", + "query": "anchor keyValuePairsReg to prevent quadratic backtracking (#6558)", + "query_tokens": [ + "anchor", + "backtracking", + "pairs", + "prevent", + "quadratic", + "value" + ], + "specific_tokens": [ + "anchor", + "backtracking", + "pairs", + "prevent", + "quadratic", + "value" + ], + "required_files": [ + "lib/content-type.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-3bcb8cfb2e5c", + "repository": "fastify", + "commit": "3bcb8cfb2e5c635a5ce60fd00f1d78ef150b91a5", + "base_revision": "c5f4a17b1342470e4de80414164723f812d095a1", + "authored_at": "2025-01-03T18:05:55+08:00", + "query": "ReadableStream.locked crashes application (#5920)", + "query_tokens": [ + "application", + "crashes", + "locked", + "readable", + "stream" + ], + "specific_tokens": [ + "application", + "crashes", + "locked", + "readable", + "stream" + ], + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "derivable_files": [ + "test/internals/errors.test.js" + ], + "incidental_files": [], + "excluded_files": [ + "docs/Reference/Errors.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-47670c1fced0", + "repository": "fastify", + "commit": "47670c1fced0dbdd630ef7719823cce74241cc63", + "base_revision": "3e04cff927534744290f26cf15220f6824e9dc66", + "authored_at": "2026-05-30T14:45:56+03:00", + "query": "include hint and docs URL in FSTWRN004 warning message (#6723)", + "query_tokens": [ + "docs", + "fstwrn004", + "hint", + "include", + "message", + "warning" + ], + "specific_tokens": [ + "docs", + "fstwrn004", + "hint", + "include", + "message", + "warning" + ], + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "docs/Reference/Warnings.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-4e1db5bd0012", + "repository": "fastify", + "commit": "4e1db5bd0012ccf63a49ff105a63e25981b9a747", + "base_revision": "a22217f9420f70017a419d8e18b2a3141ab27989", + "authored_at": "2026-03-05T12:15:57-05:00", + "query": "gate host and protocol getters on proxy trust function", + "query_tokens": [ + "function", + "gate", + "getters", + "host", + "protocol", + "proxy", + "trust" + ], + "specific_tokens": [ + "function", + "gate", + "getters", + "host", + "protocol", + "proxy", + "trust" + ], + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/trust-proxy.test.js", + "tokens": [ + "proxy", + "trust" + ] + } + ] + } + }, + { + "case_id": "fastify-4e5a3631ec73", + "repository": "fastify", + "commit": "4e5a3631ec731fa9b82689e140b399f21a8b96fd", + "base_revision": "80ef70c2541cac5cacf7e288c2db35e3763e0c09", + "authored_at": "2025-09-22T14:18:35+02:00", + "query": "correct session.close() context in http2 timeout handler (#6327)", + "query_tokens": [ + "close", + "context", + "correct", + "handler", + "http2", + "session", + "timeout" + ], + "specific_tokens": [ + "close", + "context", + "handler", + "http2", + "session", + "timeout" + ], + "required_files": [ + "lib/server.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-4fceec019ba2", + "repository": "fastify", + "commit": "4fceec019ba2f893758a3b8434cddf36cd407f4c", + "base_revision": "76d50af2e2fb281b20965e8ebb52e4a7dd4e8de0", + "authored_at": "2026-06-11T15:29:51+03:00", + "query": "correct duplicate 'not' typos in warnings.js comments (#6713)", + "query_tokens": [ + "comments", + "correct", + "duplicate", + "typos", + "warnings" + ], + "specific_tokens": [ + "comments", + "duplicate", + "typos", + "warnings" + ], + "required_files": [ + "lib/warnings.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "lib/warnings.js", + "tokens": [ + "warnings" + ] + } + ] + } + }, + { + "case_id": "fastify-4ff4c1296267", + "repository": "fastify", + "commit": "4ff4c1296267f9b004f812a9cf2b3eeedcb5bab9", + "base_revision": "76281098f57dfe9a2af4dd86576995cb772f95c2", + "authored_at": "2025-07-13T18:59:56+02:00", + "query": "account for EPIPE fetch errors in tests (#6255)", + "query_tokens": [ + "account", + "epipe", + "errors", + "fetch", + "tests" + ], + "specific_tokens": [ + "account", + "epipe", + "fetch", + "tests" + ], + "required_files": [ + "test/helper.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-55345987cb51", + "repository": "fastify", + "commit": "55345987cb51f3266a3f3ea746c2473bf9956036", + "base_revision": "e0315b9db2ac5e9eb447f95812785311b2df4eb9", + "authored_at": "2025-06-26T03:54:54+08:00", + "query": "handle abort signal in fastify.listen (#6235)", + "query_tokens": [ + "abort", + "fastify", + "handle", + "listen", + "signal" + ], + "specific_tokens": [ + "abort", + "listen", + "signal" + ], + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "derivable_files": [ + "test/server.test.js" + ], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "fastify.js", + "tokens": [ + "fastify" + ] + } + ] + } + }, + { + "case_id": "fastify-55c4841507bd", + "repository": "fastify", + "commit": "55c4841507bd796030a0aa754ed51f5865f732a4", + "base_revision": "31ef7a4bee18374f9ef9f19618d803ed0534564b", + "authored_at": "2025-06-14T15:00:47+02:00", + "query": "close pipelining flaky test (#6204)", + "query_tokens": [ + "close", + "flaky", + "pipelining", + "test" + ], + "specific_tokens": [ + "close", + "flaky", + "pipelining", + "test" + ], + "required_files": [ + "test/close-pipelining.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/close-pipelining.test.js", + "tokens": [ + "close", + "pipelining", + "test" + ] + } + ] + } + }, + { + "case_id": "fastify-6682c4f9a76c", + "repository": "fastify", + "commit": "6682c4f9a76cbb60c372ba5cad9dd2fc6e2fdb51", + "base_revision": "ab9b96eb2f93373949c253933eddb46f6772bbf4", + "authored_at": "2026-07-11T19:45:45+05:30", + "query": "normalize method in findRoute (#6838)", + "query_tokens": [ + "find", + "method", + "normalize", + "route" + ], + "specific_tokens": [ + "find", + "method", + "normalize", + "route" + ], + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "lib/route.js", + "tokens": [ + "route" + ] + }, + { + "path": "test/find-route.test.js", + "tokens": [ + "find", + "route" + ] + } + ] + } + }, + { + "case_id": "fastify-6a428d94f6bf", + "repository": "fastify", + "commit": "6a428d94f6bf42792d4b91dd7643f74f5e2960eb", + "base_revision": "79cbc96aef2a29f58b6ceff3405e744709cef004", + "authored_at": "2025-09-24T14:43:13+02:00", + "query": "accept htab ows (#6303)", + "query_tokens": [ + "accept", + "htab" + ], + "specific_tokens": [ + "accept", + "htab" + ], + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-6ac2e9537053", + "repository": "fastify", + "commit": "6ac2e953705312d0ac8776166b96f509dd153613", + "base_revision": "5f4871f931d45d576bc3385cd1e95f0d668abf4f", + "authored_at": "2026-07-05T11:20:22+02:00", + "query": "derive request.port from request.host (#6680)", + "query_tokens": [ + "derive", + "host", + "port", + "request" + ], + "specific_tokens": [ + "derive", + "host", + "port", + "request" + ], + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "derivable_files": [ + "test/internals/request.test.js" + ], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "lib/request.js", + "tokens": [ + "request" + ] + } + ] + } + }, + { + "case_id": "fastify-6d202c6203f3", + "repository": "fastify", + "commit": "6d202c6203f33ceb709dbcf6311b2d7152a8a057", + "base_revision": "10c69d153c6ee4a61878eabfb2a8f3ec34ab9c8c", + "authored_at": "2026-05-07T11:46:30+03:00", + "query": "enable diagnostics tracking for async error handlers (#6458)", + "query_tokens": [ + "async", + "diagnostics", + "enable", + "error", + "handlers", + "tracking" + ], + "specific_tokens": [ + "async", + "diagnostics", + "enable", + "handlers", + "tracking" + ], + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "lib/error-handler.js", + "tokens": [ + "error" + ] + }, + { + "path": "test/diagnostics-channel/async-error-handler.test.js", + "tokens": [ + "async", + "diagnostics", + "error" + ] + } + ] + } + }, + { + "case_id": "fastify-6e95cb9f6de4", + "repository": "fastify", + "commit": "6e95cb9f6de47deaf145138661c70a1eec34ec97", + "base_revision": "d693f43d890f07a8852b4737b048ed76458139a2", + "authored_at": "2026-08-16T16:56:42+03:00", + "query": "Fix prefix route options url (#6719)", + "query_tokens": [ + "options", + "prefix", + "route" + ], + "specific_tokens": [ + "options", + "prefix", + "route" + ], + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "lib/route.js", + "tokens": [ + "route" + ] + }, + { + "path": "test/route-prefix.test.js", + "tokens": [ + "prefix", + "route" + ] + } + ] + } + }, + { + "case_id": "fastify-6f63ce9324b7", + "repository": "fastify", + "commit": "6f63ce9324b7368ac8beff1f385b31addce35c2d", + "base_revision": "75d74e1af74598deb18f2c052d64f815a5eec8f8", + "authored_at": "2026-07-05T15:57:09+08:00", + "query": "use ContentType to detect `json` and `charset` in reply.send (#6830)", + "query_tokens": [ + "charset", + "content", + "detect", + "json", + "reply", + "send", + "type" + ], + "specific_tokens": [ + "charset", + "content", + "detect", + "json", + "reply", + "send", + "type" + ], + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "derivable_files": [ + "test/content-type.test.js", + "test/internals/reply.test.js" + ], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.6, + "shared_tokens": [ + { + "path": "lib/content-type-parser.js", + "tokens": [ + "content", + "type" + ] + }, + { + "path": "lib/content-type.js", + "tokens": [ + "content", + "type" + ] + }, + { + "path": "lib/reply.js", + "tokens": [ + "reply" + ] + } + ] + } + }, + { + "case_id": "fastify-7299a57d3fdc", + "repository": "fastify", + "commit": "7299a57d3fdce7da49f2c6d19ba08dc673e53f22", + "base_revision": "043baf988048c08ee87190193089a3b714f163fe", + "authored_at": "2026-08-03T19:07:20+05:30", + "query": "clear trailer state when removing all trailers (#6845)", + "query_tokens": [ + "clear", + "removing", + "state", + "trailer", + "trailers" + ], + "specific_tokens": [ + "clear", + "removing", + "state", + "trailer", + "trailers" + ], + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/reply-trailers.test.js", + "tokens": [ + "trailers" + ] + } + ] + } + }, + { + "case_id": "fastify-750976cc4cb0", + "repository": "fastify", + "commit": "750976cc4cb09c3aedc00b27a971601856a5eafe", + "base_revision": "e73481cc38a2c5532d022f1e796aca3b0e728e18", + "authored_at": "2025-06-05T00:39:57+02:00", + "query": "hook async flaky (#6155)", + "query_tokens": [ + "async", + "flaky", + "hook" + ], + "specific_tokens": [ + "async", + "flaky", + "hook" + ], + "required_files": [ + "test/hooks-async.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/hooks-async.test.js", + "tokens": [ + "async" + ] + } + ] + } + }, + { + "case_id": "fastify-79ab8e956482", + "repository": "fastify", + "commit": "79ab8e956482971a8cc0544ab9a846168b11998d", + "base_revision": "970c575832521fff01cc018d928d84454811173b", + "authored_at": "2025-12-23T19:21:46+01:00", + "query": "use JSON.stringify in onBadUrl for proper escaping (#6420)", + "query_tokens": [ + "escaping", + "json", + "proper", + "stringify" + ], + "specific_tokens": [ + "escaping", + "json", + "stringify" + ], + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-7f6d31980632", + "repository": "fastify", + "commit": "7f6d31980632eaa9c751dda48da8d12ff76ab411", + "base_revision": "0c28b57cef28d15b6873df77c0f76a80a0ad2269", + "authored_at": "2026-07-01T14:36:20+05:30", + "query": "clear socket._meta on reply.hijack() when onTimeout is registered (#6810)", + "query_tokens": [ + "clear", + "hijack", + "meta", + "registered", + "reply", + "socket", + "timeout" + ], + "specific_tokens": [ + "clear", + "hijack", + "meta", + "registered", + "reply", + "socket", + "timeout" + ], + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "lib/reply.js", + "tokens": [ + "reply" + ] + } + ] + } + }, + { + "case_id": "fastify-80ef70c2541c", + "repository": "fastify", + "commit": "80ef70c2541cac5cacf7e288c2db35e3763e0c09", + "base_revision": "9a67d3a4875e2c078f9f710035a7625819853677", + "authored_at": "2025-09-22T14:18:04+02:00", + "query": "(types) allow FastifySchemaValidationError[] as an error (#6326)", + "query_tokens": [ + "allow", + "error", + "fastify", + "schema", + "types", + "validation" + ], + "specific_tokens": [ + "allow", + "schema", + "types", + "validation" + ], + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/types/schema.test-d.ts", + "tokens": [ + "schema", + "types" + ] + }, + { + "path": "types/schema.d.ts", + "tokens": [ + "schema", + "types" + ] + } + ] + } + }, + { + "case_id": "fastify-812ef58f8ab1", + "repository": "fastify", + "commit": "812ef58f8ab1e765a148177d5491d6384670b19e", + "base_revision": "b84733e997340076240d93db5bbeef716df58756", + "authored_at": "2025-08-14T05:33:30-04:00", + "query": "update typescript pino type to pick (#6287)", + "query_tokens": [ + "pick", + "pino", + "type", + "typescript", + "update" + ], + "specific_tokens": [ + "pick", + "pino", + "type", + "typescript" + ], + "required_files": [ + "types/logger.d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-8435cb10f93e", + "repository": "fastify", + "commit": "8435cb10f93e149c8956f815db9e5dd59789a1d9", + "base_revision": "b01e67d46628fa8c3995c9dce79f06ccf51d845b", + "authored_at": "2025-06-17T09:13:38+02:00", + "query": "close fastify instance (#6177)", + "query_tokens": [ + "close", + "fastify", + "instance" + ], + "specific_tokens": [ + "close", + "instance" + ], + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-8b9c07b645a8", + "repository": "fastify", + "commit": "8b9c07b645a8156c23a1d2619267fc84ea879250", + "base_revision": "9d2914857906a98b6366266417d6527ab8e5e06f", + "authored_at": "2026-06-19T20:51:34+05:30", + "query": "clear socket._meta after response to prevent keep-alive leaks (#6799)", + "query_tokens": [ + "alive", + "clear", + "keep", + "leaks", + "meta", + "prevent", + "response", + "socket" + ], + "specific_tokens": [ + "alive", + "clear", + "keep", + "leaks", + "meta", + "prevent", + "response", + "socket" + ], + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-8ddd4fa85629", + "repository": "fastify", + "commit": "8ddd4fa85629b6907ef2d262c2dde0081568dee2", + "base_revision": "850ebf6ef0d71284e37c2bc98f41bbe2ec2c45c0", + "authored_at": "2025-05-11T08:58:47+02:00", + "query": "internal function _addHook failure should be turned into the rejection that app.ready() is waiting for (#6105)", + "query_tokens": [ + "failure", + "function", + "hook", + "internal", + "ready", + "rejection", + "turned", + "waiting" + ], + "specific_tokens": [ + "failure", + "function", + "hook", + "internal", + "ready", + "rejection", + "turned", + "waiting" + ], + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-8dee9be05ebf", + "repository": "fastify", + "commit": "8dee9be05ebf683cd212aeff1d294f6ea1ec405c", + "base_revision": "d457aeda8611777389c7e4713a288eb7ddb9a389", + "authored_at": "2026-03-28T23:40:11Z", + "query": "restore trustProxy function for number and string types, add null check for socketAddr (#6613)", + "query_tokens": [ + "addr", + "check", + "function", + "null", + "number", + "proxy", + "restore", + "socket", + "string", + "trust", + "types" + ], + "specific_tokens": [ + "addr", + "check", + "function", + "null", + "number", + "proxy", + "restore", + "socket", + "string", + "trust", + "types" + ], + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/trust-proxy.test.js", + "tokens": [ + "proxy", + "trust" + ] + } + ] + } + }, + { + "case_id": "fastify-9026164f5a2e", + "repository": "fastify", + "commit": "9026164f5a2e78db59ff8f98933279161a85a6e3", + "base_revision": "6d202c6203f33ceb709dbcf6311b2d7152a8a057", + "authored_at": "2026-05-07T10:47:06+02:00", + "query": "ignore duplicate trailer completions (#6714)", + "query_tokens": [ + "completions", + "duplicate", + "ignore", + "trailer" + ], + "specific_tokens": [ + "completions", + "duplicate", + "ignore", + "trailer" + ], + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-9493c0fb716c", + "repository": "fastify", + "commit": "9493c0fb716cb7eb7c3568702912e3abb92f3193", + "base_revision": "af079bd4c60c3cbebedc7640517d7288468fb5eb", + "authored_at": "2026-07-29T21:08:36+01:00", + "query": "preserve coerced root values", + "query_tokens": [ + "coerced", + "preserve", + "root", + "values" + ], + "specific_tokens": [ + "coerced", + "preserve", + "root", + "values" + ], + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/fix-root-primitive-coercion.test.js", + "tokens": [ + "root" + ] + } + ] + } + }, + { + "case_id": "fastify-970c57583252", + "repository": "fastify", + "commit": "970c575832521fff01cc018d928d84454811173b", + "base_revision": "ab9dae462a138cd0d3c2d4e4e9e691b426fbbf9e", + "authored_at": "2025-12-15T00:18:38+08:00", + "query": "set status code before publishing diagnostics error channel (#6412)", + "query_tokens": [ + "channel", + "code", + "diagnostics", + "error", + "publishing", + "status" + ], + "specific_tokens": [ + "channel", + "code", + "diagnostics", + "publishing", + "status" + ], + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "derivable_files": [ + "test/diagnostics-channel/error-status.test.js" + ], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "lib/error-handler.js", + "tokens": [ + "error" + ] + }, + { + "path": "lib/error-status.js", + "tokens": [ + "error", + "status" + ] + } + ] + } + }, + { + "case_id": "fastify-9ba6d6016fa9", + "repository": "fastify", + "commit": "9ba6d6016fa932118e6cb2a2f68617cab0824f12", + "base_revision": "74c44c94afea276256721316d25218d1cdd1aae4", + "authored_at": "2026-08-03T10:53:18+03:00", + "query": "do not force close in-flight connections on 'idle' (#6889)", + "query_tokens": [ + "close", + "connections", + "flight", + "force", + "idle" + ], + "specific_tokens": [ + "close", + "connections", + "flight", + "force", + "idle" + ], + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/close.test.js", + "tokens": [ + "close" + ] + } + ] + } + }, + { + "case_id": "fastify-9bbf60978d01", + "repository": "fastify", + "commit": "9bbf60978d013c26663b7be8031ecc12e6db12bd", + "base_revision": "efb599a7ac2b4e5ec148cf6a2f547b6dd3ea4eb7", + "authored_at": "2026-04-23T03:20:35-07:00", + "query": "allow request.getValidationFunction() to return undefined (#6665)", + "query_tokens": [ + "allow", + "function", + "request", + "return", + "undefined", + "validation" + ], + "specific_tokens": [ + "allow", + "function", + "request", + "return", + "undefined", + "validation" + ], + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/types/request.test-d.ts", + "tokens": [ + "request" + ] + }, + { + "path": "types/request.d.ts", + "tokens": [ + "request" + ] + } + ] + } + }, + { + "case_id": "fastify-9be9fc07b769", + "repository": "fastify", + "commit": "9be9fc07b7697a4818699e26effe3364d9d3ce4a", + "base_revision": "b84061926bb9f7afda65104e1d8d29abbd64090e", + "authored_at": "2025-07-30T09:55:03+02:00", + "query": "remove unnecessary body length check in contentTypeParser (#6266)", + "query_tokens": [ + "body", + "check", + "content", + "length", + "parser", + "remove", + "type", + "unnecessary" + ], + "specific_tokens": [ + "body", + "check", + "content", + "length", + "parser", + "remove", + "type", + "unnecessary" + ], + "required_files": [ + "lib/contentTypeParser.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "lib/contentTypeParser.js", + "tokens": [ + "content", + "parser", + "type" + ] + } + ] + } + }, + { + "case_id": "fastify-9dc6486b6594", + "repository": "fastify", + "commit": "9dc6486b659420bb213fe93fde2432f74e1905ec", + "base_revision": "7868e47194bcfb34cab2e51ea36bf10f764b538e", + "authored_at": "2025-03-30T19:19:39+09:00", + "query": "clear `[kState].readyPromise` for garbage collection (#6030)", + "query_tokens": [ + "clear", + "collection", + "garbage", + "promise", + "ready", + "state" + ], + "specific_tokens": [ + "clear", + "collection", + "garbage", + "promise", + "ready", + "state" + ], + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-9eaef5123b88", + "repository": "fastify", + "commit": "9eaef5123b8824ac64e2b30692e78320d3bb8987", + "base_revision": "39e87e892da468bbffd4aeb4360c92dca22f4b53", + "authored_at": "2026-08-06T13:26:27+05:00", + "query": "pass null instead of undefined to requestCompleted on success (#6837)", + "query_tokens": [ + "completed", + "instead", + "null", + "pass", + "request", + "success", + "undefined" + ], + "specific_tokens": [ + "completed", + "instead", + "null", + "pass", + "request", + "success", + "undefined" + ], + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-af079bd4c60c", + "repository": "fastify", + "commit": "af079bd4c60c3cbebedc7640517d7288468fb5eb", + "base_revision": "6e95cb9f6de47deaf145138661c70a1eec34ec97", + "authored_at": "2026-07-06T18:25:03+02:00", + "query": "disable numeric trustProxy hop-count trust", + "query_tokens": [ + "count", + "disable", + "numeric", + "proxy", + "trust" + ], + "specific_tokens": [ + "count", + "disable", + "numeric", + "proxy", + "trust" + ], + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "docs/Reference/Server.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.25, + "shared_tokens": [ + { + "path": "test/trust-proxy.test.js", + "tokens": [ + "proxy", + "trust" + ] + } + ] + } + }, + { + "case_id": "fastify-b03d0783f246", + "repository": "fastify", + "commit": "b03d0783f24602748ca5042cc30dfadefdd2c53a", + "base_revision": "6a428d94f6bf42792d4b91dd7643f74f5e2960eb", + "authored_at": "2025-09-26T00:12:11+02:00", + "query": "handle non FastifyErrors in custom handler properly, set type of error-parameter for setErrorHandler and errorHandler to unknown, but configurable via generic TError (#6308)", + "query_tokens": [ + "configurable", + "custom", + "error", + "errors", + "fastify", + "generic", + "handle", + "handler", + "parameter", + "properly", + "terror", + "type", + "unknown" + ], + "specific_tokens": [ + "configurable", + "custom", + "generic", + "handler", + "parameter", + "terror", + "type", + "unknown" + ], + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.25, + "shared_tokens": [ + { + "path": "lib/error-handler.js", + "tokens": [ + "error", + "handler" + ] + } + ] + } + }, + { + "case_id": "fastify-b0e255a2cdab", + "repository": "fastify", + "commit": "b0e255a2cdab05ed382a99db70fde482f6464bdd", + "base_revision": "0d7182129610148dda7f996f1a0962c43842a228", + "authored_at": "2025-08-10T16:01:11+02:00", + "query": "OPTIONS Content-Type handling (#6263)", + "query_tokens": [ + "content", + "handling", + "options", + "type" + ], + "specific_tokens": [ + "content", + "options", + "type" + ], + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-b4db85b665ac", + "repository": "fastify", + "commit": "b4db85b665ac99a809c633b7564d49c8a4c2c5c9", + "base_revision": "3653e7b92719061c6c2dfb4497d2d5b936509309", + "authored_at": "2026-02-20T23:08:12+01:00", + "query": "avoid mutating shared routerOptions across instances (#6515)", + "query_tokens": [ + "across", + "avoid", + "instances", + "mutating", + "options", + "router", + "shared" + ], + "specific_tokens": [ + "across", + "avoid", + "instances", + "mutating", + "options", + "router", + "shared" + ], + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/router-options.test.js", + "tokens": [ + "options", + "router" + ] + } + ] + } + }, + { + "case_id": "fastify-c02c5f5696c8", + "repository": "fastify", + "commit": "c02c5f5696c8695184f355891106ec26139cb559", + "base_revision": "f18cda12d62a81f41d14b5c41cd674dd2abef817", + "authored_at": "2026-01-03T18:14:22+08:00", + "query": "require send() payload when Reply type is specified (#6432)", + "query_tokens": [ + "payload", + "reply", + "require", + "send", + "specified", + "type" + ], + "specific_tokens": [ + "payload", + "reply", + "require", + "send", + "specified", + "type" + ], + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.8, + "shared_tokens": [ + { + "path": "test/types/reply.test-d.ts", + "tokens": [ + "reply" + ] + }, + { + "path": "test/types/type-provider.test-d.ts", + "tokens": [ + "type" + ] + }, + { + "path": "types/reply.d.ts", + "tokens": [ + "reply" + ] + }, + { + "path": "types/type-provider.d.ts", + "tokens": [ + "type" + ] + } + ] + } + }, + { + "case_id": "fastify-c06fc06d33e6", + "repository": "fastify", + "commit": "c06fc06d33e66354cf6f5dc000b1ef2002a5eb96", + "base_revision": "d1133d7eac09e075ccfd3b947f6a6e25d727d4d4", + "authored_at": "2026-04-21T22:01:17+02:00", + "query": "use ContentType parser for response schema lookup (#6685)", + "query_tokens": [ + "content", + "lookup", + "parser", + "response", + "schema", + "type" + ], + "specific_tokens": [ + "content", + "lookup", + "parser", + "response", + "schema", + "type" + ], + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/schema-serialization.test.js", + "tokens": [ + "schema" + ] + } + ] + } + }, + { + "case_id": "fastify-c277b9f0ec27", + "repository": "fastify", + "commit": "c277b9f0ec27356de6551d7777117bd739057e99", + "base_revision": "95292098426943822fcbc40beaacb314db4bbfbd", + "authored_at": "2025-07-20T18:36:53+01:00", + "query": "add FST_ERR_CTP_INVALID_JSON_BODY (#5925)", + "query_tokens": [ + "body", + "invalid", + "json" + ], + "specific_tokens": [ + "body", + "invalid", + "json" + ], + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "derivable_files": [ + "test/internals/errors.test.js" + ], + "incidental_files": [], + "excluded_files": [ + "docs/Reference/Errors.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-c2ba18c17769", + "repository": "fastify", + "commit": "c2ba18c17769dacce61472ec0397560b3b4eebde", + "base_revision": "d33f017ffef3a9beec687b4d4b1777e2e1180afa", + "authored_at": "2026-04-19T05:58:31-07:00", + "query": "avoid duplicate closeIdleConnections call on native servers (#6669)", + "query_tokens": [ + "avoid", + "call", + "close", + "connections", + "duplicate", + "idle", + "native", + "servers" + ], + "specific_tokens": [ + "avoid", + "call", + "close", + "connections", + "duplicate", + "idle", + "native", + "servers" + ], + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "test/close-pipelining.test.js", + "tokens": [ + "close" + ] + } + ] + } + }, + { + "case_id": "fastify-c4a6e78e5d77", + "repository": "fastify", + "commit": "c4a6e78e5d7727b0d4323be75623f7815869b5da", + "base_revision": "b4db85b665ac99a809c633b7564d49c8a4c2c5c9", + "authored_at": "2026-02-20T23:08:27+01:00", + "query": "accept async route hooks in shorthand options (#6514)", + "query_tokens": [ + "accept", + "async", + "hooks", + "options", + "route", + "shorthand" + ], + "specific_tokens": [ + "accept", + "async", + "hooks", + "options", + "route", + "shorthand" + ], + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/types/route.test-d.ts", + "tokens": [ + "route" + ] + }, + { + "path": "types/route.d.ts", + "tokens": [ + "route" + ] + } + ] + } + }, + { + "case_id": "fastify-c647de44ccea", + "repository": "fastify", + "commit": "c647de44cceaf8ec6ce81796374ff6ffd944d873", + "base_revision": "27f4d6222dd945238e96c1fdaa632d5dc3c37b2c", + "authored_at": "2026-06-09T15:27:33+03:00", + "query": "replace AssertionError with FST_ERR_PLUGIN_DEPENDENCY_NOT_REGISTERED in checkDependencies (#6774)", + "query_tokens": [ + "assertion", + "check", + "dependencies", + "dependency", + "error", + "plugin", + "registered", + "replace" + ], + "specific_tokens": [ + "assertion", + "check", + "dependencies", + "dependency", + "plugin", + "registered", + "replace" + ], + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "derivable_files": [ + "test/internals/errors.test.js" + ], + "incidental_files": [], + "excluded_files": [ + "docs/Reference/Errors.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.4, + "shared_tokens": [ + { + "path": "lib/plugin-utils.js", + "tokens": [ + "plugin" + ] + }, + { + "path": "test/internals/plugin.test.js", + "tokens": [ + "plugin" + ] + } + ] + } + }, + { + "case_id": "fastify-cc8d9a3c961d", + "repository": "fastify", + "commit": "cc8d9a3c961d65cb1a39c4142ce63285d2de1e3c", + "base_revision": "6e6be153127cbe6e025f73efba78d1db7dd5788b", + "authored_at": "2026-06-28T00:45:40-07:00", + "query": "make hasRequestDecorator/hasReplyDecorator recognize constructor-assigned built-in properties (#6753)", + "query_tokens": [ + "assigned", + "built", + "constructor", + "decorator", + "make", + "properties", + "recognize", + "reply", + "request" + ], + "specific_tokens": [ + "assigned", + "built", + "constructor", + "decorator", + "make", + "properties", + "recognize", + "reply", + "request" + ], + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.75, + "shared_tokens": [ + { + "path": "lib/reply.js", + "tokens": [ + "reply" + ] + }, + { + "path": "lib/request.js", + "tokens": [ + "request" + ] + }, + { + "path": "test/decorator-instance-properties.test.js", + "tokens": [ + "decorator", + "properties" + ] + } + ] + } + }, + { + "case_id": "fastify-d338dca5ab24", + "repository": "fastify", + "commit": "d338dca5ab24a4ce0175b4333efe46859ceaffab", + "base_revision": "e1aee4b872aaeabbd4f7074794c1015a518f8d9a", + "authored_at": "2025-11-08T15:16:10+01:00", + "query": "consistent error handling for custom validators in async validation contexts (#6247)", + "query_tokens": [ + "async", + "consistent", + "contexts", + "custom", + "error", + "handling", + "validation", + "validators" + ], + "specific_tokens": [ + "async", + "consistent", + "contexts", + "custom", + "validation", + "validators" + ], + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "lib/validation.js", + "tokens": [ + "validation" + ] + }, + { + "path": "test/validation-error-handling.test.js", + "tokens": [ + "error", + "handling", + "validation" + ] + } + ] + } + }, + { + "case_id": "fastify-d33f017ffef3", + "repository": "fastify", + "commit": "d33f017ffef3a9beec687b4d4b1777e2e1180afa", + "base_revision": "dce0d47a44ecf6605a4e6b39d80f0bef17c2c2ce", + "authored_at": "2026-04-19T15:51:05+08:00", + "query": "prevent duplicate res.end in sendTrailer with sync callbacks (#6676)", + "query_tokens": [ + "callbacks", + "duplicate", + "prevent", + "send", + "sync", + "trailer" + ], + "specific_tokens": [ + "callbacks", + "duplicate", + "prevent", + "send", + "sync", + "trailer" + ], + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-d44b3f7b2d2e", + "repository": "fastify", + "commit": "d44b3f7b2d2e89ef17fe514f52cb0dff74124ac0", + "base_revision": "7d7f065e0efb1b2d9f3ba318ac3915a1ac59d5c6", + "authored_at": "2025-08-15T18:01:29-04:00", + "query": "Fix use of \"esModuleInterop: false\" (#6292)", + "query_tokens": [ + "false", + "interop", + "module" + ], + "specific_tokens": [ + "false", + "interop", + "module" + ], + "required_files": [ + "types/logger.d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-d76dbcd58bf6", + "repository": "fastify", + "commit": "d76dbcd58bf6d82bba167cdc74a79d2f74dbcb97", + "base_revision": "d9659819fbbc391d054f1aa4b49ce969a13610dd", + "authored_at": "2026-04-21T01:37:32+08:00", + "query": "correct isCustomSerializerCompiler flag check (#6657)", + "query_tokens": [ + "check", + "compiler", + "correct", + "custom", + "flag", + "serializer" + ], + "specific_tokens": [ + "check", + "compiler", + "custom", + "flag", + "serializer" + ], + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-d9659819fbbc", + "repository": "fastify", + "commit": "d9659819fbbc391d054f1aa4b49ce969a13610dd", + "base_revision": "c2ba18c17769dacce61472ec0397560b3b4eebde", + "authored_at": "2026-04-19T19:55:38+02:00", + "query": "error.code not present on some routing errors (#6674) (#6678)", + "query_tokens": [ + "code", + "error", + "errors", + "present", + "routing" + ], + "specific_tokens": [ + "code", + "present", + "routing" + ], + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-dd02e428ddbf", + "repository": "fastify", + "commit": "dd02e428ddbfe407a2d114d431e60a6a10335ea0", + "base_revision": "810e3d548eeceb32c948a0fbfdd90d6d1e6098ce", + "authored_at": "2025-11-03T19:42:10+09:00", + "query": "handle web stream payload in HEAD route (#6372)", + "query_tokens": [ + "handle", + "head", + "payload", + "route", + "stream" + ], + "specific_tokens": [ + "head", + "payload", + "route", + "stream" + ], + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "lib/head-route.js", + "tokens": [ + "head", + "route" + ] + }, + { + "path": "test/route.6.test.js", + "tokens": [ + "route" + ] + }, + { + "path": "test/route.7.test.js", + "tokens": [ + "route" + ] + } + ] + } + }, + { + "case_id": "fastify-e1aee4b872aa", + "repository": "fastify", + "commit": "e1aee4b872aaeabbd4f7074794c1015a518f8d9a", + "base_revision": "b5958b3b8ed6fce850ebaabbc4a8a3e5f42b389b", + "authored_at": "2025-11-05T08:24:31+01:00", + "query": "parse ipv6 hostname (#6373)", + "query_tokens": [ + "hostname", + "ipv6", + "parse" + ], + "specific_tokens": [ + "hostname", + "ipv6", + "parse" + ], + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-e1f58aba76e0", + "repository": "fastify", + "commit": "e1f58aba76e0510cd69e4e69b507598253b70070", + "base_revision": "387b168997acbffb82ea9b10377a43b21aceb259", + "authored_at": "2025-10-07T21:49:55+11:00", + "query": "respect child logger factory in fastify options (#6349)", + "query_tokens": [ + "child", + "factory", + "fastify", + "logger", + "options", + "respect" + ], + "specific_tokens": [ + "child", + "factory", + "logger", + "options", + "respect" + ], + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "fastify.js", + "tokens": [ + "fastify" + ] + }, + { + "path": "test/child-logger-factory.test.js", + "tokens": [ + "child", + "factory", + "logger" + ] + } + ] + } + }, + { + "case_id": "fastify-e4ffc205328d", + "repository": "fastify", + "commit": "e4ffc205328db294d550c5855d2573b33f5e9d62", + "base_revision": "ed664c344f4d7e0171f252d5e886db6d7d241770", + "authored_at": "2026-08-12T16:02:56+05:30", + "query": "remove raw response headers (#6860)", + "query_tokens": [ + "headers", + "remove", + "response" + ], + "specific_tokens": [ + "headers", + "remove", + "response" + ], + "required_files": [ + "lib/reply.js" + ], + "derivable_files": [ + "test/internals/reply.test.js" + ], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-f18cda12d62a", + "repository": "fastify", + "commit": "f18cda12d62a81f41d14b5c41cd674dd2abef817", + "base_revision": "4c10980bd4a5a4d19e051f52a018779e6afb588b", + "authored_at": "2026-01-02T22:09:58+05:30", + "query": "Align routerOptions defaultRoute types with runtime (#6392)", + "query_tokens": [ + "align", + "default", + "options", + "route", + "router", + "runtime", + "types" + ], + "specific_tokens": [ + "align", + "default", + "options", + "route", + "router", + "runtime", + "types" + ], + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "docs/Reference/Server.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "test/router-options.test.js", + "tokens": [ + "options", + "router" + ] + }, + { + "path": "test/types/instance.test-d.ts", + "tokens": [ + "types" + ] + } + ] + } + }, + { + "case_id": "fastify-f3d2bcb3963c", + "repository": "fastify", + "commit": "f3d2bcb3963cd570a582e5d39aab01a9ae692fe4", + "base_revision": "9b1cf06987c1b00c90e27ed921a2779edd2f609b", + "authored_at": "2025-04-18T22:23:54+02:00", + "query": "treat space as a delimiter in content-type parsing (#6064)", + "query_tokens": [ + "content", + "delimiter", + "parsing", + "space", + "treat", + "type" + ], + "specific_tokens": [ + "content", + "delimiter", + "parsing", + "space", + "treat", + "type" + ], + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "fastify-f5ef34443c5b", + "repository": "fastify", + "commit": "f5ef34443c5be0cfc7fbde44f84253ef5d0ec66e", + "base_revision": "9a1535bcfe2a780cc6f9c6555df86e5072ca9a29", + "authored_at": "2026-08-14T14:20:56+05:00", + "query": "run preClose hook exactly once per declaring instance (#6940)", + "query_tokens": [ + "close", + "declaring", + "exactly", + "hook", + "instance", + "once" + ], + "specific_tokens": [ + "close", + "declaring", + "exactly", + "hook", + "instance", + "once" + ], + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/close.test.js", + "tokens": [ + "close" + ] + } + ] + } + } + ], + "rejected_sample": [ + { + "commit": "83e69762aff047940b376fc2a266d143499ef727", + "subject": "test(types): assert LogController.requestCompleted accepts an undefined error (#6875)", + "reason": "not-a-fix" + }, + { + "commit": "dbebe73c421597da96c3154c1036377f4e8e5117", + "subject": "docs: fix broken links in Style-Guide.md (#6966)", + "reason": "excluded-subject" + }, + { + "commit": "9334d07129589f7340adfa4729ebe6c55e34bbfd", + "subject": "Bumped v6.0.0-alpha.2", + "reason": "not-a-fix" + }, + { + "commit": "d693f43d890f07a8852b4737b048ed76458139a2", + "subject": "docs: remove deprecated-API notices ahead of v6 cleanup (#6951)", + "reason": "excluded-subject" + }, + { + "commit": "9a1535bcfe2a780cc6f9c6555df86e5072ca9a29", + "subject": "ci: allow backport in pr title (#6948)", + "reason": "not-a-fix" + }, + { + "commit": "b7de40faa21e3cff3a214921223dff2203944902", + "subject": "test: fix reply.mediaType (#6949)", + "reason": "not-a-fix" + }, + { + "commit": "377533a260d5de8966f4f9dbe7de9eff5e0d0da9", + "subject": "Bumped v6.0.0-alpha.1", + "reason": "not-a-fix" + }, + { + "commit": "eaff726a68c629c1aedc99f96b3c4b95dfdb38d1", + "subject": "feat: add `Reply.prototype.mediaType` (#6932)", + "reason": "not-a-fix" + }, + { + "commit": "3efe8748335fea67b7006773a8ea28dd4abc8724", + "subject": "test: use process-warning spyWarning for testing (#6887)", + "reason": "not-a-fix" + }, + { + "commit": "ed664c344f4d7e0171f252d5e886db6d7d241770", + "subject": "Bumped v6.0.0-alpha.0", + "reason": "not-a-fix" + }, + { + "commit": "2808d207d93c5b56d5ed9cf02ada65c15bb6e8f8", + "subject": "refactor(types)!: remove ResSerializerReply type (#6911)", + "reason": "not-a-fix" + }, + { + "commit": "42321d896471b4a6790c5d7ad619c5586c4a283c", + "subject": "refactor(types)!: remove deprecated req in request.d.ts (#6935)", + "reason": "not-a-fix" + }, + { + "commit": "e1fc82593b614a991e6d237ce2c134f7bf8f9254", + "subject": "test: remove @sinonjs/fake-timers (#6934)", + "reason": "not-a-fix" + }, + { + "commit": "bd3e246de0c107bb9b1f87666a67efd36dbd5da1", + "subject": "refactor!: remove FSTDEP025 (#6909)", + "reason": "not-a-fix" + }, + { + "commit": "63ef53497c52c3b4fdb8c5e0c1b6b14dd5a424fc", + "subject": "refactor!: remove FSTDEP024 deprecation (#6912)", + "reason": "not-a-fix" + }, + { + "commit": "7df35fb8b4238d64086479c3f4caba28f24e418f", + "subject": "refactor!: remove FSTDEP023 deprecation (#6913)", + "reason": "not-a-fix" + }, + { + "commit": "2e81c38bfede1858c3705c5a6f7753abd93019b3", + "subject": "docs: add per-route logging for Google Cloud Functions (#6907)", + "reason": "excluded-subject" + }, + { + "commit": "f87eb61a6df212f56849e6ba250f505648aeb8a1", + "subject": "chore(sponsor): add testmu ai (#6922)", + "reason": "excluded-subject" + }, + { + "commit": "a22ceded3dfbb44354dcb0fe683d4b142a7487c4", + "subject": "Bumped v5.11.3", + "reason": "not-a-fix" + }, + { + "commit": "b8cb861f155edf559e618af818f6b8f486ceb090", + "subject": "refactor!: remove fstdep022 (#6908)", + "reason": "not-a-fix" + } + ] +} diff --git a/benchmarks/context-retrieval/corpora/corpus-flask.json b/benchmarks/context-retrieval/corpora/corpus-flask.json new file mode 100644 index 00000000..09fc64f5 --- /dev/null +++ b/benchmarks/context-retrieval/corpora/corpus-flask.json @@ -0,0 +1,1350 @@ +{ + "schema_version": "codevetter.context-retrieval-corpus.v1", + "repository": { + "id": "flask", + "head": "d318b683471101618febed18996405ad26462110" + }, + "selection": { + "commits_scanned": 600, + "fixes_only": true, + "max_required_files": 8 + }, + "counts": { + "cases": 39, + "path_leak": 7, + "baseline_blind": 32, + "multi_file": 17 + }, + "cases": [ + { + "case_id": "flask-03b410066b96", + "repository": "flask", + "commit": "03b410066b96f63ba2a215750f881d5bebb78a29", + "base_revision": "4696156278c5d3f05fb09d1a605b6866b49ffbcb", + "authored_at": "2022-07-11T18:02:00-04:00", + "query": "Fix docs/CHANGES.rst typo", + "query_tokens": [ + "changes", + "docs", + "typo" + ], + "specific_tokens": [ + "docs", + "typo" + ], + "required_files": [ + "CHANGES.rst" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "CHANGES.rst", + "tokens": [ + "changes" + ] + } + ] + } + }, + { + "case_id": "flask-095651be9eec", + "repository": "flask", + "commit": "095651be9eec58ddb0c2eb6158318b1c703c67c5", + "base_revision": "02a08512527f997e43d1c01d03d4d473cab3b239", + "authored_at": "2022-07-25T07:35:17-07:00", + "query": "fix has_app_context and has_request_context", + "query_tokens": [ + "context", + "request" + ], + "specific_tokens": [ + "context", + "request" + ], + "required_files": [ + "src/flask/ctx.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-0afeb1d11cad", + "repository": "flask", + "commit": "0afeb1d11cad9aa42ef25cc20b10dcd67c8cf3c9", + "base_revision": "4cf8b78c6c674048a393acf0280f08761a5cb2a5", + "authored_at": "2023-05-01T09:01:40-07:00", + "query": "fix release date", + "query_tokens": [ + "date", + "release" + ], + "specific_tokens": [ + "date", + "release" + ], + "required_files": [ + "CHANGES.rst" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-0d8c8ba71bc6", + "repository": "flask", + "commit": "0d8c8ba71bc6362e6ea9af08146dc97e1a0a8abc", + "base_revision": "9c3deeee963e62dfb60c9343fad4ec2ec752fdba", + "authored_at": "2022-09-18T19:53:47+08:00", + "query": "Fix docstring of test_request_context (#4821)", + "query_tokens": [ + "context", + "docstring", + "request", + "test" + ], + "specific_tokens": [ + "context", + "docstring", + "request", + "test" + ], + "required_files": [ + "src/flask/app.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-0ec9192cf25f", + "repository": "flask", + "commit": "0ec9192cf25f5187d6521f2539489fa8d55336bb", + "base_revision": "345f18442ceb29f9e365af99fc85bdc5986323d2", + "authored_at": "2023-04-25T21:10:05+02:00", + "query": "fix importing Markup from flask", + "query_tokens": [ + "flask", + "importing", + "markup" + ], + "specific_tokens": [ + "importing", + "markup" + ], + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "src/flask/__init__.py", + "tokens": [ + "flask" + ] + } + ] + } + }, + { + "case_id": "flask-176fdfa00023", + "repository": "flask", + "commit": "176fdfa0002360670f698d3828cfff11f97349f0", + "base_revision": "2d31dce8262a68875c177645fa0c7585a45ac447", + "authored_at": "2024-08-23T16:33:52-07:00", + "query": "fix mypy findings", + "query_tokens": [ + "findings", + "mypy" + ], + "specific_tokens": [ + "findings", + "mypy" + ], + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-1af8f95785b8", + "repository": "flask", + "commit": "1af8f95785b85d0937153ee63d504c0b51934d80", + "base_revision": "3435d2ff1589eb0c1a85cc294a20985910a1a606", + "authored_at": "2024-01-26T10:08:55+08:00", + "query": "fix super call in list comprehension", + "query_tokens": [ + "call", + "comprehension", + "list", + "super" + ], + "specific_tokens": [ + "call", + "comprehension", + "list", + "super" + ], + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-1d5abfadd713", + "repository": "flask", + "commit": "1d5abfadd7132c9a78e14e5ba6c07aed47115280", + "base_revision": "b97165db75c6f4e99c3307b4a5a1f3b0d9f4de25", + "authored_at": "2023-11-24T16:03:31-06:00", + "query": "Fixing issue 5342: 'The double quote is missing in the string'", + "query_tokens": [ + "double", + "fixing", + "issue", + "missing", + "quote", + "string" + ], + "specific_tokens": [ + "double", + "fixing", + "missing", + "quote", + "string" + ], + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-261e4a6cf287", + "repository": "flask", + "commit": "261e4a6cf287180b69c4db407791e43ce90e50ad", + "base_revision": "8f13f5b6d672f1ba434f9c8ebe2c1b1dd385962e", + "authored_at": "2023-01-18T10:31:08-08:00", + "query": "fix flake8 bugbear errors", + "query_tokens": [ + "bugbear", + "errors", + "flake8" + ], + "specific_tokens": [ + "bugbear", + "flake8" + ], + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-3ba37d2afe65", + "repository": "flask", + "commit": "3ba37d2afe6511c3f3153248f7342174bea5b131", + "base_revision": "88bcf78439b66b5fcba9f4d3a1c56307f98dbf1d", + "authored_at": "2022-06-06T08:24:05-07:00", + "query": "fix uninstalled package tests under tox", + "query_tokens": [ + "package", + "tests", + "uninstalled" + ], + "specific_tokens": [ + "package", + "tests", + "uninstalled" + ], + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "tests/test_instance_config.py", + "tokens": [ + "tests" + ] + } + ] + } + }, + { + "case_id": "flask-40b78fa2ea90", + "repository": "flask", + "commit": "40b78fa2ea9095197608287de9f0d902d2763b00", + "base_revision": "2c5d652493b79eecadd4407f24f2249948bd6ff2", + "authored_at": "2024-08-23T16:41:39-07:00", + "query": "fix min python for pip-compile", + "query_tokens": [ + "compile", + "python" + ], + "specific_tokens": [ + "compile", + "python" + ], + "required_files": [ + "tox.ini" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "requirements/build.txt", + "requirements/dev.txt", + "requirements/docs.txt", + "requirements/tests.txt", + "requirements/typing.txt" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-426a1e25b77e", + "repository": "flask", + "commit": "426a1e25b77e760b4f54bf94aee3e3617850569f", + "base_revision": "6f7d99ce4b7dc2bad52683e64fec7008dd0b580f", + "authored_at": "2022-02-08T12:05:17-08:00", + "query": "fix pytest 7 warnings", + "query_tokens": [ + "pytest", + "warnings" + ], + "specific_tokens": [ + "pytest", + "warnings" + ], + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-4995a775df21", + "repository": "flask", + "commit": "4995a775df21a206b529403bc30d71795a994fd4", + "base_revision": "07c7d5730a2685ef2281cc635e289685e5c3d478", + "authored_at": "2024-11-10T20:13:05-08:00", + "query": "fix subdomain_matching=False behavior", + "query_tokens": [ + "behavior", + "false", + "matching", + "subdomain" + ], + "specific_tokens": [ + "behavior", + "false", + "matching", + "subdomain" + ], + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "docs/config.rst" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-54c3f87af9f6", + "repository": "flask", + "commit": "54c3f87af9f6c0d6622cec2c3d9695b0229fb8fa", + "base_revision": "ea08f155d8d37d515559534bbe0f80ef5185466c", + "authored_at": "2024-11-23T03:23:09+03:00", + "query": "fix type hint for `cli_runner.invoke`", + "query_tokens": [ + "hint", + "invoke", + "runner", + "type" + ], + "specific_tokens": [ + "hint", + "invoke", + "runner", + "type" + ], + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-6f79cb8a2346", + "repository": "flask", + "commit": "6f79cb8a23467aa01f94f6ad53790f468ba77c76", + "base_revision": "b6a8ccd2cf65939401fe4ccb6b550d2b19423b06", + "authored_at": "2022-02-16T21:12:01-08:00", + "query": "Fix typing on `app.session_interface`", + "query_tokens": [ + "interface", + "session", + "typing" + ], + "specific_tokens": [ + "interface", + "session", + "typing" + ], + "required_files": [ + "src/flask/app.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-7203feabf723", + "repository": "flask", + "commit": "7203feabf723edae0286ae5dc64fec8ac4c91735", + "base_revision": "de8429ffda8cfb54db175529e2bae72a24e1fa7e", + "authored_at": "2026-07-14T16:42:09+02:00", + "query": "Fix IPv6 server name parsing", + "query_tokens": [ + "ipv6", + "name", + "parsing", + "server" + ], + "specific_tokens": [ + "ipv6", + "name", + "parsing", + "server" + ], + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-746455d1038f", + "repository": "flask", + "commit": "746455d1038fd5479f396cbf2d19b48edfeb8290", + "base_revision": "e0c157f7ee99bde3376e456a914bb07260eb6017", + "authored_at": "2022-08-14T18:59:19-04:00", + "query": "Fix misrendered docstring", + "query_tokens": [ + "docstring", + "misrendered" + ], + "specific_tokens": [ + "docstring", + "misrendered" + ], + "required_files": [ + "src/flask/config.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-74721b48f034", + "repository": "flask", + "commit": "74721b48f0347f19f134f386fab3421e5132fb53", + "base_revision": "d273f87a41e72db8ef1387f469ffa1c6eed39730", + "authored_at": "2024-10-22T17:32:02-03:00", + "query": "Fix the issue link in the Flask 3.0.1 Changelog in the send_file argument type entry", + "query_tokens": [ + "argument", + "changelog", + "entry", + "file", + "flask", + "issue", + "link", + "send", + "type" + ], + "specific_tokens": [ + "argument", + "changelog", + "entry", + "file", + "link", + "send", + "type" + ], + "required_files": [ + "CHANGES.rst" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-80cf589a26d7", + "repository": "flask", + "commit": "80cf589a26d792820f135591eb8007bca891bf4f", + "base_revision": "cc80a47f5be6c60e53c42ef9378e6db3b0e183b1", + "authored_at": "2023-08-14T11:10:15+01:00", + "query": "Correct the error handler typing", + "query_tokens": [ + "correct", + "error", + "handler", + "typing" + ], + "specific_tokens": [ + "handler", + "typing" + ], + "required_files": [ + "src/flask/typing.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "src/flask/typing.py", + "tokens": [ + "typing" + ] + } + ] + } + }, + { + "case_id": "flask-826514b8eb18", + "repository": "flask", + "commit": "826514b8eb18f6c314cf566630253d35c89e42c3", + "base_revision": "6d266f63633f5f127165d4ef836db14a59bbc106", + "authored_at": "2023-08-16T13:37:56-07:00", + "query": "fix flake8 bugbear findings", + "query_tokens": [ + "bugbear", + "findings", + "flake8" + ], + "specific_tokens": [ + "bugbear", + "findings", + "flake8" + ], + "required_files": [ + "tests/test_basic.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-860a25c390eb", + "repository": "flask", + "commit": "860a25c390eba8e6c089a818b02800dd9d789864", + "base_revision": "273123f6b8fc0d871553f707e515816777c60e24", + "authored_at": "2024-05-06T10:23:10-07:00", + "query": "fix mypy finding", + "query_tokens": [ + "finding", + "mypy" + ], + "specific_tokens": [ + "finding", + "mypy" + ], + "required_files": [ + "src/flask/helpers.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-9532cba45d23", + "repository": "flask", + "commit": "9532cba45d2339e90ebf04f178b1e4f2064e7328", + "base_revision": "0bc7356ce1ae11e633426902aba76d525f4523da", + "authored_at": "2023-05-01T08:10:25-07:00", + "query": "fix mypy finding", + "query_tokens": [ + "finding", + "mypy" + ], + "specific_tokens": [ + "finding", + "mypy" + ], + "required_files": [ + "src/flask/app.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-976459f7cb35", + "repository": "flask", + "commit": "976459f7cb35c8c3abc3a6e6a2728d76d1a0a5f2", + "base_revision": "5e621a2801daf95be6d208da9dcc287bb251f63c", + "authored_at": "2026-02-03T10:20:49-08:00", + "query": "fix editable werkzeug", + "query_tokens": [ + "editable", + "werkzeug" + ], + "specific_tokens": [ + "editable", + "werkzeug" + ], + "required_files": [ + "pyproject.toml" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "uv.lock" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-98ae71897600", + "repository": "flask", + "commit": "98ae71897600a3162f529b1b30c70b8abdf0961b", + "base_revision": "c62b03bcfd6e6440f8195e02f4678488e16121ac", + "authored_at": "2024-11-01T18:06:34-07:00", + "query": "fix mypy finding", + "query_tokens": [ + "finding", + "mypy" + ], + "specific_tokens": [ + "finding", + "mypy" + ], + "required_files": [ + "src/flask/app.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-9b74a90dd3c4", + "repository": "flask", + "commit": "9b74a90dd3c47f792734823e8793ac36f38bc4dd", + "base_revision": "5880befcd22490c40f3c63d6178427f7753672c1", + "authored_at": "2026-01-24T19:11:02-08:00", + "query": "fix codespell findings", + "query_tokens": [ + "codespell", + "findings" + ], + "specific_tokens": [ + "codespell", + "findings" + ], + "required_files": [ + ".pre-commit-config.yaml" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "docs/appcontext.rst", + "docs/quickstart.rst", + "docs/templating.rst" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-a363642a3288", + "repository": "flask", + "commit": "a363642a32880114c7944b1017f2f69bd4cc00eb", + "base_revision": "f958b6500b2aeb8edc578a07f00546becdad4c80", + "authored_at": "2024-05-06T10:09:58-07:00", + "query": "fix mypy finding with new werkzeug endpoint type", + "query_tokens": [ + "endpoint", + "finding", + "mypy", + "type", + "werkzeug" + ], + "specific_tokens": [ + "endpoint", + "finding", + "mypy", + "type", + "werkzeug" + ], + "required_files": [ + "src/flask/wrappers.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-a6a7a57380cd", + "repository": "flask", + "commit": "a6a7a57380cd8f7410753c3b819ba6d09198d8c9", + "base_revision": "4984753dbf5a247f46e2903011c981d0709973ff", + "authored_at": "2022-08-04T07:24:20-07:00", + "query": "fix default value of app.env", + "query_tokens": [ + "default", + "value" + ], + "specific_tokens": [ + "default", + "value" + ], + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-bda295d37fa3", + "repository": "flask", + "commit": "bda295d37fa3809c95a93d01dd605d0222109889", + "base_revision": "c8cf4694c60f0d81809468a1b45ec730496cc546", + "authored_at": "2023-06-09T09:34:42-07:00", + "query": "fix use of importlib.util.find_spec", + "query_tokens": [ + "find", + "importlib", + "spec", + "util" + ], + "specific_tokens": [ + "find", + "importlib", + "spec", + "util" + ], + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-c49ce2e1eb8b", + "repository": "flask", + "commit": "c49ce2e1eb8b598e724388bead0943837bab8027", + "base_revision": "3237fff4b86227a84913d3885b600738bb9479a0", + "authored_at": "2023-08-16T13:37:26-07:00", + "query": "fix flake8 bugbear findings", + "query_tokens": [ + "bugbear", + "findings", + "flake8" + ], + "specific_tokens": [ + "bugbear", + "findings", + "flake8" + ], + "required_files": [ + "tests/test_basic.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-ccf125bf3010", + "repository": "flask", + "commit": "ccf125bf30102beedafb3ecef72376bfb6c90a4d", + "base_revision": "db0fe9436e9da7f7bd14683542bbf33ba3c6d5e4", + "authored_at": "2024-04-23T16:49:42-07:00", + "query": "fix mypy findings", + "query_tokens": [ + "findings", + "mypy" + ], + "specific_tokens": [ + "findings", + "mypy" + ], + "required_files": [ + "src/flask/sessions.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-d7b6c1f6703d", + "repository": "flask", + "commit": "d7b6c1f6703df405c69da45e7e0ba3d1aed512ce", + "base_revision": "fa1ee7066807c21256e90089731c548b313394d2", + "authored_at": "2022-10-31T12:49:16+04:00", + "query": "Fix subdomain inheritance for nested blueprints.", + "query_tokens": [ + "blueprints", + "inheritance", + "nested", + "subdomain" + ], + "specific_tokens": [ + "blueprints", + "inheritance", + "nested", + "subdomain" + ], + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "src/flask/blueprints.py", + "tokens": [ + "blueprints" + ] + }, + { + "path": "tests/test_blueprints.py", + "tokens": [ + "blueprints" + ] + } + ] + } + }, + { + "case_id": "flask-de8429ffda8c", + "repository": "flask", + "commit": "de8429ffda8cfb54db175529e2bae72a24e1fa7e", + "base_revision": "514fc6b3e8402e4c646d5284e97a4f0ab50a7c4b", + "authored_at": "2026-07-14T16:40:58+02:00", + "query": "Fix IPv6 session transactions", + "query_tokens": [ + "ipv6", + "session", + "transactions" + ], + "specific_tokens": [ + "ipv6", + "session", + "transactions" + ], + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-df201ed152b6", + "repository": "flask", + "commit": "df201ed152b61c439ea1ace179f35b424594ba6c", + "base_revision": "96800fb673cb7b2d75476096798e701e3e6d26bc", + "authored_at": "2024-11-01T16:44:17-07:00", + "query": "fix js example test", + "query_tokens": [ + "example", + "test" + ], + "specific_tokens": [ + "example", + "test" + ], + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "examples/javascript/tests/test_js_example.py", + "tokens": [ + "example", + "test" + ] + } + ] + } + }, + { + "case_id": "flask-dffe3034825e", + "repository": "flask", + "commit": "dffe3034825edc4f84aded013fc9c6590e545ed5", + "base_revision": "c5a5576522273ff028018c4e13d1fc4d0260495b", + "authored_at": "2024-10-18T13:04:35-07:00", + "query": "fix mypy findings", + "query_tokens": [ + "findings", + "mypy" + ], + "specific_tokens": [ + "findings", + "mypy" + ], + "required_files": [ + "src/flask/helpers.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-e63ead4208c1", + "repository": "flask", + "commit": "e63ead4208c1e56723532878728ef9c2eb197b01", + "base_revision": "c77b099cbb979269d45df06753cadd8abc662e23", + "authored_at": "2024-09-22T21:42:35+08:00", + "query": "Fix typo in the changelog", + "query_tokens": [ + "changelog", + "typo" + ], + "specific_tokens": [ + "changelog", + "typo" + ], + "required_files": [ + "CHANGES.rst" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-e82db2ca3a22", + "repository": "flask", + "commit": "e82db2ca3a22c9614c1987392c9cfaa8c6ce99ad", + "base_revision": "d3b78fd18a8d9e224cb9ef58a23cec9b1ffc9ce9", + "authored_at": "2026-02-12T13:03:03-08:00", + "query": "fix provide_automatic_options override", + "query_tokens": [ + "automatic", + "options", + "override", + "provide" + ], + "specific_tokens": [ + "automatic", + "options", + "override", + "provide" + ], + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "docs/config.rst" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-eb1182a10e75", + "repository": "flask", + "commit": "eb1182a10e757b5e76009c69fdf3a88a5afc6840", + "base_revision": "a363642a32880114c7944b1017f2f69bd4cc00eb", + "authored_at": "2024-05-06T10:12:32-07:00", + "query": "fix mypy finding", + "query_tokens": [ + "finding", + "mypy" + ], + "specific_tokens": [ + "finding", + "mypy" + ], + "required_files": [ + "src/flask/sessions.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-ecc057dd4857", + "repository": "flask", + "commit": "ecc057dd4857e655f0844b781899a917852042a4", + "base_revision": "3207af8827d4162ef5841249c46fbb9da043c551", + "authored_at": "2024-01-23T11:28:51-05:00", + "query": "fix jinja_loader annotation", + "query_tokens": [ + "annotation", + "jinja", + "loader" + ], + "specific_tokens": [ + "annotation", + "jinja", + "loader" + ], + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "flask-eede1a3685e2", + "repository": "flask", + "commit": "eede1a3685e21deaeb6686e9de9b76a73b6a510c", + "base_revision": "69f71b4d94006678a2463f76a4504208d195c367", + "authored_at": "2022-04-12T13:17:24-03:00", + "query": "fix annotation for json.loads", + "query_tokens": [ + "annotation", + "json", + "loads" + ], + "specific_tokens": [ + "annotation", + "json", + "loads" + ], + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "src/flask/json/__init__.py", + "tokens": [ + "json" + ] + } + ] + } + } + ], + "rejected_sample": [ + { + "commit": "d318b683471101618febed18996405ad26462110", + "subject": "explain seek", + "reason": "not-a-fix" + }, + { + "commit": "2a8a38b051fc248865730bf3511bf2e2ea325e81", + "subject": "support query in methodview", + "reason": "not-a-fix" + }, + { + "commit": "89992954ec71b594b1b911f98977cdc8ad46a057", + "subject": "add app.query route decorator", + "reason": "not-a-fix" + }, + { + "commit": "3596b1ab61cea85edb8970e83ff61daa073facf8", + "subject": "add entry", + "reason": "not-a-fix" + }, + { + "commit": "81798b406f97ffe4795d2c9fc18db7db744a96dc", + "subject": "fix typos", + "reason": "query-too-short" + }, + { + "commit": "8b4fd5d18611e63080b826ecfefe8659dba7d2e4", + "subject": "use header properties", + "reason": "not-a-fix" + }, + { + "commit": "30d1c5d6fcbd62d9e902209d7a207f7510e9f5dc", + "subject": "test Python 3.15", + "reason": "not-a-fix" + }, + { + "commit": "ff6fa22f4d961a28567db11430208b964620f511", + "subject": "update dependencies", + "reason": "not-a-fix" + }, + { + "commit": "5ce121dd841e2dc2594d0ed3954278f4101d3521", + "subject": "Replace the use of private `monkeypatch` fixture API", + "reason": "not-a-fix" + }, + { + "commit": "8285adf5a4f572dbf7a0a3b8f3a96ca0b19c50a4", + "subject": "update dev dependencies", + "reason": "not-a-fix" + }, + { + "commit": "36e4a824f340fdee7ed50937ba8e7f6bc7d17f81", + "subject": "Any for CertParamType type", + "reason": "not-a-fix" + }, + { + "commit": "954f5684e4841aad84a8eec7ace7b81a0d3f6831", + "subject": "update dev dependencies", + "reason": "not-a-fix" + }, + { + "commit": "1d49747264554dee31f5627d4249d09aaaeced39", + "subject": "update flask-mongoengine link", + "reason": "not-a-fix" + }, + { + "commit": "7374c85ddefc3f4b177a698ab9f0cbb6a5c0b392", + "subject": "remove leftover setuptools", + "reason": "not-a-fix" + }, + { + "commit": "9368fb3f3c52d74534d14c1bef03c79c103356cd", + "subject": "case-insensitive comparison", + "reason": "not-a-fix" + }, + { + "commit": "06ea505ce2b2042af26e96d35ebf159af7c0869d", + "subject": "separate copy per call", + "reason": "not-a-fix" + }, + { + "commit": "689362089edd09b6d68f7cfe99075e1345e0fede", + "subject": "fix typo", + "reason": "query-too-short" + }, + { + "commit": "a31e6b73469cb2bf7eb8f70b5ff21f710fd2e23c", + "subject": "remove werkzeug host tests", + "reason": "not-a-fix" + }, + { + "commit": "b21425d6df207fec0c47e9563faa10a2819984ca", + "subject": "reduce venv size", + "reason": "not-a-fix" + }, + { + "commit": "83dbcb222a65a87741c9d96bb183f34528149269", + "subject": "update dev dependencies", + "reason": "not-a-fix" + } + ] +} diff --git a/benchmarks/context-retrieval/corpora/corpus-gin.json b/benchmarks/context-retrieval/corpora/corpus-gin.json new file mode 100644 index 00000000..0bbd55d3 --- /dev/null +++ b/benchmarks/context-retrieval/corpora/corpus-gin.json @@ -0,0 +1,2153 @@ +{ + "schema_version": "codevetter.context-retrieval-corpus.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "selection": { + "commits_scanned": 400, + "fixes_only": true, + "max_required_files": 8 + }, + "counts": { + "cases": 56, + "path_leak": 18, + "baseline_blind": 38, + "multi_file": 43 + }, + "cases": [ + { + "case_id": "gin-09f8224593e3", + "repository": "gin", + "commit": "09f8224593e31edf3c58ab3f13bc31ef53473733", + "base_revision": "f75144a356e57c95bd21a048f0a40492dcdb33c5", + "authored_at": "2024-03-06T14:16:53+05:30", + "query": "Add fullPath in context copy (#3784)", + "query_tokens": [ + "context", + "copy", + "full", + "path" + ], + "specific_tokens": [ + "context", + "copy", + "full", + "path" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context.go", + "tokens": [ + "context" + ] + }, + { + "path": "context_test.go", + "tokens": [ + "context" + ] + } + ] + } + }, + { + "case_id": "gin-212267d67163", + "repository": "gin", + "commit": "212267d6716324536be15512c0b6ed080794b798", + "base_revision": "b682b8a54e165088efa71d8b941e7ad28cde348b", + "authored_at": "2022-11-08T19:54:48+08:00", + "query": "fix typo in comment (#3371)", + "query_tokens": [ + "comment", + "typo" + ], + "specific_tokens": [ + "comment", + "typo" + ], + "required_files": [ + "context.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-234a6d4c00cb", + "repository": "gin", + "commit": "234a6d4c00cb77af9852aca0b8289745d5529b4b", + "base_revision": "df2753778e7bc5c2dd559361cf0c97b2b313e9bb", + "authored_at": "2025-09-26T08:13:39+08:00", + "query": "refine hijack behavior for response lifecycle (#4373)", + "query_tokens": [ + "behavior", + "hijack", + "lifecycle", + "refine", + "response" + ], + "specific_tokens": [ + "behavior", + "hijack", + "lifecycle", + "refine", + "response" + ], + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "response_writer.go", + "tokens": [ + "response" + ] + }, + { + "path": "response_writer_test.go", + "tokens": [ + "response" + ] + } + ] + } + }, + { + "case_id": "gin-24a1d2adb9db", + "repository": "gin", + "commit": "24a1d2adb9dba64d38426d76f2a4cf76123a4711", + "base_revision": "4c64f1c3859fa853659ff1db65a8f0fa67856ed9", + "authored_at": "2022-10-16T11:41:14+10:00", + "query": "spelling `covert` -> `convert` (#3325)", + "query_tokens": [ + "convert", + "covert", + "spelling" + ], + "specific_tokens": [ + "convert", + "covert", + "spelling" + ], + "required_files": [ + "binding/form_mapping.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-28e57f58b184", + "repository": "gin", + "commit": "28e57f58b184b2305ace192e02496bb89f6fd8cb", + "base_revision": "3cb30679b5e3021db16c776ed7e70d380586e9ce", + "authored_at": "2024-09-06T08:21:19+03:00", + "query": "Set default value for form fields (#4047)", + "query_tokens": [ + "default", + "fields", + "form", + "value" + ], + "specific_tokens": [ + "default", + "fields", + "form", + "value" + ], + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/form_mapping.go", + "tokens": [ + "form" + ] + }, + { + "path": "binding/form_mapping_test.go", + "tokens": [ + "form" + ] + } + ] + } + }, + { + "case_id": "gin-293ad7edebb3", + "repository": "gin", + "commit": "293ad7edebb3ae30369288bd6416ca0d78474727", + "base_revision": "2e4d4f38962a6f15ae496d59b294f307eef95429", + "authored_at": "2026-06-22T14:39:26+03:30", + "query": "Copy() copies Errors and Accepted fields (#4695)", + "query_tokens": [ + "accepted", + "copies", + "copy", + "errors", + "fields" + ], + "specific_tokens": [ + "accepted", + "copies", + "copy", + "fields" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-2a794cd0b0fa", + "repository": "gin", + "commit": "2a794cd0b0faa7d829291375b27a3467ea972b0d", + "base_revision": "b917b14ff9d189f16a7492be79d123a47806ee19", + "authored_at": "2025-12-04T11:49:37+09:00", + "query": "version mismatch (#4403)", + "query_tokens": [ + "mismatch", + "version" + ], + "specific_tokens": [ + "mismatch", + "version" + ], + "required_files": [ + "debug.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-2ae61570499d", + "repository": "gin", + "commit": "2ae61570499d8bb5eb05e46d22a3754cf2635e63", + "base_revision": "fb13e822a4b48b872dd4538dd5bbd7a70767ca7a", + "authored_at": "2022-08-31T01:34:33-05:00", + "query": "Fix typos in RouterGroup method docs (#3302)", + "query_tokens": [ + "docs", + "group", + "method", + "router", + "typos" + ], + "specific_tokens": [ + "docs", + "group", + "method", + "router", + "typos" + ], + "required_files": [ + "routergroup.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-2b1da2b0b38d", + "repository": "gin", + "commit": "2b1da2b0b38dfc5d5841266037c0c8b249eca1dd", + "base_revision": "0d9dbbb44551a872d30fd89d4d55ba0515d646fd", + "authored_at": "2024-03-21T21:08:41+08:00", + "query": "make context Value method adhere to Go standards (#3897)", + "query_tokens": [ + "adhere", + "context", + "make", + "method", + "standards", + "value" + ], + "specific_tokens": [ + "adhere", + "context", + "make", + "method", + "standards", + "value" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context.go", + "tokens": [ + "context" + ] + }, + { + "path": "context_test.go", + "tokens": [ + "context" + ] + } + ] + } + }, + { + "case_id": "gin-2d4bbec94155", + "repository": "gin", + "commit": "2d4bbec941551479b1fdf1e54ece03e6e82a7e72", + "base_revision": "9f5ecd4be440f2789db917aa93c1b8afa2276917", + "authored_at": "2023-05-29T10:57:53+09:00", + "query": "fix lack of escaping of filename in Content-Disposition (#3556)", + "query_tokens": [ + "content", + "disposition", + "escaping", + "filename", + "lack" + ], + "specific_tokens": [ + "content", + "disposition", + "escaping", + "filename", + "lack" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-36b0dede4b8c", + "repository": "gin", + "commit": "36b0dede4b8c4a67d92c4107cebc5a068364321d", + "base_revision": "3f5b0afa2ac85ea79638ca08f4140ce64b8246e5", + "authored_at": "2024-05-13T14:55:41+08:00", + "query": "check handler is nil (#3413)", + "query_tokens": [ + "check", + "handler" + ], + "specific_tokens": [ + "check", + "handler" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-386d244068db", + "repository": "gin", + "commit": "386d244068db3693f938db4ead6d1f5f85942e3f", + "base_revision": "44d0dd70924dd154e3b98bc340accc53484efa9c", + "authored_at": "2023-12-07T00:38:55Z", + "query": "correctly expand the capacity of params (#3502)", + "query_tokens": [ + "capacity", + "correctly", + "expand", + "params" + ], + "specific_tokens": [ + "capacity", + "expand", + "params" + ], + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-3dc1cd6572b4", + "repository": "gin", + "commit": "3dc1cd6572b4e3a0cd170a15debe546c2c72294f", + "base_revision": "9f598a31aafb92d675f38f1c8371e4ac76f858bf", + "authored_at": "2024-02-05T10:46:35+08:00", + "query": "binding error while not upload file (#3819) (#3820)", + "query_tokens": [ + "binding", + "error", + "file", + "upload" + ], + "specific_tokens": [ + "binding", + "file", + "upload" + ], + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/form_mapping.go", + "tokens": [ + "binding" + ] + }, + { + "path": "binding/form_mapping_test.go", + "tokens": [ + "binding" + ] + } + ] + } + }, + { + "case_id": "gin-44d0dd70924d", + "repository": "gin", + "commit": "44d0dd70924dd154e3b98bc340accc53484efa9c", + "base_revision": "49f45a542719df661bd71dd48f1595f0bc1ff6f7", + "authored_at": "2023-11-16T21:16:43+05:30", + "query": "Add pointer support for url query params (#3659) (#3666)", + "query_tokens": [ + "params", + "pointer", + "query", + "support" + ], + "specific_tokens": [ + "params", + "pointer", + "query", + "support" + ], + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-472d086af2ac", + "repository": "gin", + "commit": "472d086af2acd924cb4b9d7be0525f7d790f69bc", + "base_revision": "fb2583442c4d9bccb75e6d26f1aa6e7c01950db6", + "authored_at": "2026-02-27T19:15:27-08:00", + "query": "panic in findCaseInsensitivePathRec with RedirectFixedPath (#4535)", + "query_tokens": [ + "case", + "find", + "fixed", + "insensitive", + "panic", + "path", + "redirect" + ], + "specific_tokens": [ + "case", + "find", + "fixed", + "insensitive", + "panic", + "path", + "redirect" + ], + "required_files": [ + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-4a3eb31fb15b", + "repository": "gin", + "commit": "4a3eb31fb15b2a2d78b4bdbe0c31a2c564b1977a", + "base_revision": "293ad7edebb3ae30369288bd6416ca0d78474727", + "authored_at": "2026-06-22T20:56:28+08:00", + "query": "record recovered panics in c.Errors (#4698)", + "query_tokens": [ + "errors", + "panics", + "record", + "recovered" + ], + "specific_tokens": [ + "panics", + "record", + "recovered" + ], + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-4a40f8f1a49b", + "repository": "gin", + "commit": "4a40f8f1a49b9086b461d97e167c3b9628d8b923", + "base_revision": "857db39f82fb82456af2906ccea972ae1d65ff57", + "authored_at": "2024-02-01T09:00:17+08:00", + "query": "upgrade golang.org/x/crypto to 0.17.0 (#3832)", + "query_tokens": [ + "crypto", + "golang", + "upgrade" + ], + "specific_tokens": [ + "crypto", + "golang", + "upgrade" + ], + "required_files": [ + "go.mod", + "go.sum" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-4cee78f5382d", + "repository": "gin", + "commit": "4cee78f5382d5245c3652e6c15fee715eec505c3", + "base_revision": "fc1c43298de675e5252d0b44f97dc5e204bd4e1e", + "authored_at": "2023-02-19T22:25:48+09:00", + "query": "Fix #3500 Add escape logic for header (#3503)", + "query_tokens": [ + "escape", + "header", + "logic" + ], + "specific_tokens": [ + "escape", + "header", + "logic" + ], + "required_files": [ + "gin.go", + "routes_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-4f339e6a35b1", + "repository": "gin", + "commit": "4f339e6a35b163d31b30916b37f4176d385f41bd", + "base_revision": "36b0dede4b8c4a67d92c4107cebc5a068364321d", + "authored_at": "2024-05-14T10:25:54+08:00", + "query": "YAML judgment logic in Negotiate (#3966)", + "query_tokens": [ + "judgment", + "logic", + "negotiate", + "yaml" + ], + "specific_tokens": [ + "judgment", + "logic", + "negotiate", + "yaml" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-51aea73ba0f1", + "repository": "gin", + "commit": "51aea73ba0f125f6cacc3b4b695efdf21d9c634f", + "base_revision": "33ab0fc155e68acbc7fd30281c0d9b2e6e091b7b", + "authored_at": "2022-10-20T00:49:19+08:00", + "query": "modify interface check way (#3327)", + "query_tokens": [ + "check", + "interface", + "modify" + ], + "specific_tokens": [ + "check", + "interface", + "modify" + ], + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-55e27f12465e", + "repository": "gin", + "commit": "55e27f12465e058058180280d5f0bdc473eb3302", + "base_revision": "971fe21876e491f2597a390522adc849272e3bec", + "authored_at": "2022-11-06T17:08:11+08:00", + "query": "missing route params for CreateTestContext (#2778) (#2803)", + "query_tokens": [ + "context", + "create", + "missing", + "params", + "route", + "test" + ], + "specific_tokens": [ + "context", + "create", + "missing", + "params", + "route", + "test" + ], + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.667, + "shared_tokens": [ + { + "path": "context_test.go", + "tokens": [ + "context", + "test" + ] + }, + { + "path": "test_helpers.go", + "tokens": [ + "test" + ] + } + ] + } + }, + { + "case_id": "gin-5826722a87cf", + "repository": "gin", + "commit": "5826722a87cf5855fcc4b794cbef11612352771d", + "base_revision": "bdc1ad7987b6931801af9d50e2df25667fdfaaf4", + "authored_at": "2025-07-17T20:51:40+09:00", + "query": "version number discrepancy (#4299)", + "query_tokens": [ + "discrepancy", + "number", + "version" + ], + "specific_tokens": [ + "discrepancy", + "number", + "version" + ], + "required_files": [ + "debug.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-5c00df8afadd", + "repository": "gin", + "commit": "5c00df8afadd06cc5be530dde00fe6d9fa4a2e4a", + "base_revision": "db309081bc5c137b2aa15701ef53f7f19788da25", + "authored_at": "2026-02-28T09:07:31+07:00", + "query": "write content length in Data.Render (#4206)", + "query_tokens": [ + "content", + "data", + "length", + "render", + "write" + ], + "specific_tokens": [ + "content", + "data", + "length", + "render", + "write" + ], + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "render/data.go", + "tokens": [ + "data", + "render" + ] + }, + { + "path": "render/render_test.go", + "tokens": [ + "render" + ] + } + ] + } + }, + { + "case_id": "gin-5fad976b372e", + "repository": "gin", + "commit": "5fad976b372e381312f8de69f0969f1284d229d3", + "base_revision": "93ff771e6dbf10e432864b30f3719ac5c84a4d4a", + "authored_at": "2025-11-16T06:52:07+05:30", + "query": "literal colon routes not working with engine.Handler() (#4415)", + "query_tokens": [ + "colon", + "engine", + "handler", + "literal", + "routes", + "working" + ], + "specific_tokens": [ + "colon", + "engine", + "handler", + "literal", + "routes", + "working" + ], + "required_files": [ + "gin.go", + "gin_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-626d55b0c029", + "repository": "gin", + "commit": "626d55b0c02937645c21774cacc021713de88604", + "base_revision": "9c081de9cdd1948f521d47d170d18cbc2981c33a", + "authored_at": "2024-06-22T16:19:04+02:00", + "query": "Do not panic when handling method not allowed on empty tree (#4003)", + "query_tokens": [ + "allowed", + "empty", + "handling", + "method", + "panic", + "tree" + ], + "specific_tokens": [ + "allowed", + "empty", + "method", + "panic", + "tree" + ], + "required_files": [ + "gin.go", + "gin_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-63dd3e60cab8", + "repository": "gin", + "commit": "63dd3e60cab89c27fb66bce1423bd268d52abad1", + "base_revision": "c358d5656d0feb8b310d4ec379bccde46ccc8cc7", + "authored_at": "2025-11-27T23:20:52+08:00", + "query": "suppress http.ErrAbortHandler in recover (#4336)", + "query_tokens": [ + "abort", + "handler", + "http", + "recover", + "suppress" + ], + "specific_tokens": [ + "abort", + "handler", + "http", + "recover", + "suppress" + ], + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-646312aef6a3", + "repository": "gin", + "commit": "646312aef6a34095476ac846b0920db5fb24b2ea", + "base_revision": "5f458dd1a6d631f324e4af9a4f5429ffdf199342", + "authored_at": "2024-03-11T22:24:36+08:00", + "query": "protect Context.Keys map when call Copy method (#3873)", + "query_tokens": [ + "call", + "context", + "copy", + "keys", + "method", + "protect" + ], + "specific_tokens": [ + "call", + "context", + "copy", + "keys", + "method", + "protect" + ], + "required_files": [ + "context.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context.go", + "tokens": [ + "context" + ] + } + ] + } + }, + { + "case_id": "gin-674522db91d6", + "repository": "gin", + "commit": "674522db91d637d179c16c372d87756ea26fa089", + "base_revision": "8f7c340577e19245827f7ba71ef3e0143cc7eeee", + "authored_at": "2025-05-21T19:21:46+08:00", + "query": "binding time with empty value (#4103)", + "query_tokens": [ + "binding", + "empty", + "time", + "value" + ], + "specific_tokens": [ + "binding", + "empty", + "time", + "value" + ], + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/form_mapping.go", + "tokens": [ + "binding" + ] + }, + { + "path": "binding/form_mapping_test.go", + "tokens": [ + "binding" + ] + } + ] + } + }, + { + "case_id": "gin-7a1b655074c3", + "repository": "gin", + "commit": "7a1b655074c313f9491c83bb8ea164cdc4a9afe9", + "base_revision": "67c9d4ee110e9adfe33063ef847dba56717c148a", + "authored_at": "2025-05-11T20:04:09+05:30", + "query": "sonic on arm64 (#4234)", + "query_tokens": [ + "arm64", + "sonic" + ], + "specific_tokens": [ + "arm64", + "sonic" + ], + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + ".github/workflows/gin.yml", + "docs/doc.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.25, + "shared_tokens": [ + { + "path": "internal/json/sonic.go", + "tokens": [ + "sonic" + ] + } + ] + } + }, + { + "case_id": "gin-7cb151bb4c4c", + "repository": "gin", + "commit": "7cb151bb4c4cfc6018a00a125422ff38a041b9f8", + "base_revision": "3010cbd7f4eccdbb610c510274895e083b8c058c", + "authored_at": "2023-01-16T15:50:07+01:00", + "query": "panic on NegotiateFormat - index out of range (#3397)", + "query_tokens": [ + "format", + "index", + "negotiate", + "panic", + "range" + ], + "specific_tokens": [ + "format", + "index", + "negotiate", + "panic", + "range" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-7d147928ee23", + "repository": "gin", + "commit": "7d147928ee232fce156ea7ce8ae6329e148aeb41", + "base_revision": "f5f5da8fa09d12a22225c493fa8191fb14bdd5bf", + "authored_at": "2024-05-08T04:13:36+03:00", + "query": "data race warning for gin mode (#1580)", + "query_tokens": [ + "data", + "mode", + "race", + "warning" + ], + "specific_tokens": [ + "data", + "mode", + "race", + "warning" + ], + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.667, + "shared_tokens": [ + { + "path": "mode.go", + "tokens": [ + "mode" + ] + }, + { + "path": "mode_test.go", + "tokens": [ + "mode" + ] + } + ] + } + }, + { + "case_id": "gin-814cd188eb0b", + "repository": "gin", + "commit": "814cd188eb0b0304dd3c37b698edc51ff46e24a2", + "base_revision": "2c9e5fe47ae55defc55dae30a0e63192e4538bc2", + "authored_at": "2022-09-18T16:59:57+03:00", + "query": "FIX TYPO: Gin by default useR -> ... useS (#3324)", + "query_tokens": [ + "default", + "typo" + ], + "specific_tokens": [ + "default", + "typo" + ], + "required_files": [ + "routergroup.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-82bcd6d39bfe", + "repository": "gin", + "commit": "82bcd6d39bfe9c22032764ff3b0b6f8ef1673e49", + "base_revision": "86ff4a64c7efe1a1c875529835eeef9e15de1e86", + "authored_at": "2024-02-07T06:44:11-05:00", + "query": "dereference pointer to struct (#3199)", + "query_tokens": [ + "dereference", + "pointer", + "struct" + ], + "specific_tokens": [ + "dereference", + "pointer", + "struct" + ], + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-86ff4a64c7ef", + "repository": "gin", + "commit": "86ff4a64c7efe1a1c875529835eeef9e15de1e86", + "base_revision": "e957d1abf13846e458956d8c97e7b7c76c7ee9a3", + "authored_at": "2024-02-06T04:08:56+01:00", + "query": "Allow header according to RFC 7231 (HTTP 405) (#3759)", + "query_tokens": [ + "according", + "allow", + "header", + "http" + ], + "specific_tokens": [ + "according", + "allow", + "header", + "http" + ], + "required_files": [ + "gin.go", + "routes_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-8763f33c65f7", + "repository": "gin", + "commit": "8763f33c65f7df8be5b9fe7504ab7fcf20abb41d", + "base_revision": "e737e3e267beb4dc3bab16cc8be59e3902d98a94", + "authored_at": "2025-03-20T17:40:41+02:00", + "query": "prevent middleware re-entry issue in HandleContext (#3987)", + "query_tokens": [ + "context", + "entry", + "handle", + "issue", + "middleware", + "prevent" + ], + "specific_tokens": [ + "context", + "entry", + "middleware", + "prevent" + ], + "required_files": [ + "gin.go", + "gin_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-8790d08909fc", + "repository": "gin", + "commit": "8790d08909fc4d193c6c787c9c72f3089168f411", + "base_revision": "78f4687875d72d10392f8a77008cbefdec4c0aa0", + "authored_at": "2024-03-21T16:28:42+02:00", + "query": "query binding bug (#3236)", + "query_tokens": [ + "binding", + "query" + ], + "specific_tokens": [ + "binding", + "query" + ], + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/query.go", + "tokens": [ + "binding", + "query" + ] + }, + { + "path": "binding/query_test.go", + "tokens": [ + "binding", + "query" + ] + } + ] + } + }, + { + "case_id": "gin-8b9c55e8b059", + "repository": "gin", + "commit": "8b9c55e8b059a1ffcc69e3b5dc0b9d583958811f", + "base_revision": "51aea73ba0f125f6cacc3b4b695efdf21d9c634f", + "authored_at": "2022-11-06T17:02:40+08:00", + "query": "redirectSlash bug (#3227)", + "query_tokens": [ + "redirect", + "slash" + ], + "specific_tokens": [ + "redirect", + "slash" + ], + "required_files": [ + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-8dd20118ba9a", + "repository": "gin", + "commit": "8dd20118ba9a9dfa18ebba99e33d6a59b0090436", + "base_revision": "00cfe5aac2604e0632ee674b69a1e652a2461923", + "authored_at": "2026-08-15T11:13:07+05:30", + "query": "bump golang.org/x/net and golang.org/x/text to patched versions (#4807)", + "query_tokens": [ + "bump", + "golang", + "patched", + "text", + "versions" + ], + "specific_tokens": [ + "bump", + "golang", + "patched", + "text", + "versions" + ], + "required_files": [ + "go.mod", + "go.sum" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-8e07d37c63e5", + "repository": "gin", + "commit": "8e07d37c63e5536eb25f4af4c91eabeee4011fba", + "base_revision": "d7776de7d444935ea4385999711bd6331a98fecb", + "authored_at": "2026-02-13T11:39:14+05:45", + "query": "Correct typos, improve documentation clarity, and remove dead code (#4511)", + "query_tokens": [ + "clarity", + "code", + "correct", + "dead", + "documentation", + "improve", + "remove", + "typos" + ], + "specific_tokens": [ + "clarity", + "code", + "dead", + "documentation", + "remove", + "typos" + ], + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "CONTRIBUTING.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-8eb5f832bac1", + "repository": "gin", + "commit": "8eb5f832bac1853fc84c508a2b9406134d39492e", + "base_revision": "c58e0d59ca6753da47daa0a64b844bc869030759", + "authored_at": "2023-01-07T01:57:54+01:00", + "query": "tree bug where loop index is not decremented. (#3460)", + "query_tokens": [ + "decremented", + "index", + "loop", + "tree" + ], + "specific_tokens": [ + "decremented", + "index", + "loop", + "tree" + ], + "required_files": [ + "routes_test.go", + "tree.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "tree.go", + "tokens": [ + "tree" + ] + } + ] + } + }, + { + "case_id": "gin-9914178584e4", + "repository": "gin", + "commit": "9914178584e42458ff7d23891463a880f58c9d86", + "base_revision": "915e4c90d28ec4cffc6eb146e208ab5a65eac772", + "authored_at": "2026-01-02T02:15:27Z", + "query": "ClientIP handling for multiple X-Forwarded-For header values (#4472)", + "query_tokens": [ + "client", + "forwarded", + "handling", + "header", + "multiple", + "values" + ], + "specific_tokens": [ + "client", + "forwarded", + "header", + "multiple", + "values" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-9f598a31aafb", + "repository": "gin", + "commit": "9f598a31aafb92d675f38f1c8371e4ac76f858bf", + "base_revision": "c6ae2e69666a2b36203b29650ee75d172c725c66", + "authored_at": "2024-02-04T18:44:29+05:30", + "query": "catch-all conflicting wildcard (#3812)", + "query_tokens": [ + "catch", + "conflicting", + "wildcard" + ], + "specific_tokens": [ + "conflicting", + "wildcard" + ], + "required_files": [ + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-aefae309a4fc", + "repository": "gin", + "commit": "aefae309a4fc197ce5d57cd8391562b6d2a63a95", + "base_revision": "8edb7a71a17061bbaa85db53bb8ba6daad3c3aa8", + "authored_at": "2022-11-06T17:12:11+08:00", + "query": "test fmt.Println replace t.Error (#3328)", + "query_tokens": [ + "error", + "println", + "replace", + "test" + ], + "specific_tokens": [ + "println", + "replace", + "test" + ], + "required_files": [ + "gin_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "gin_test.go", + "tokens": [ + "test" + ] + } + ] + } + }, + { + "case_id": "gin-b38c59de7fef", + "repository": "gin", + "commit": "b38c59de7fef67400a1c98efeae700a689c45783", + "base_revision": "cf32d2dcf8c7534f59727c5e213e45f2412c593a", + "authored_at": "2025-05-11T18:38:33+04:00", + "query": "change Unwrap method receiver to value type (#4232)", + "query_tokens": [ + "change", + "method", + "receiver", + "type", + "unwrap", + "value" + ], + "specific_tokens": [ + "method", + "receiver", + "type", + "unwrap", + "value" + ], + "required_files": [ + "errors.go", + "errors_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-b917b14ff9d1", + "repository": "gin", + "commit": "b917b14ff9d189f16a7492be79d123a47806ee19", + "base_revision": "fad706f1216e6d12bdd51d28d5a40ec27e6c6453", + "authored_at": "2025-12-03T19:18:10+08:00", + "query": "empty value error (#2169)", + "query_tokens": [ + "empty", + "error", + "value" + ], + "specific_tokens": [ + "empty", + "value" + ], + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-bb1fc2e0fe97", + "repository": "gin", + "commit": "bb1fc2e0fe97c63dab1527baab88d01183853b8f", + "base_revision": "2d4bbec941551479b1fdf1e54ece03e6e82a7e72", + "authored_at": "2023-05-29T01:59:35Z", + "query": "fix Request.Context() checks (#3512)", + "query_tokens": [ + "checks", + "context", + "request" + ], + "specific_tokens": [ + "checks", + "context", + "request" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context.go", + "tokens": [ + "context" + ] + }, + { + "path": "context_test.go", + "tokens": [ + "context" + ] + } + ] + } + }, + { + "case_id": "gin-c3d1092b3b48", + "repository": "gin", + "commit": "c3d1092b3b48addf6f9cd00fe274ec3bd14650eb", + "base_revision": "9968c4bf9d5a99edc3eee2c068a4c9160ece8915", + "authored_at": "2025-10-11T19:20:41+08:00", + "query": "improve empty slice/array handling in form binding (#4380)", + "query_tokens": [ + "array", + "binding", + "empty", + "form", + "handling", + "improve", + "slice" + ], + "specific_tokens": [ + "array", + "binding", + "empty", + "form", + "slice" + ], + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/form_mapping.go", + "tokens": [ + "binding", + "form" + ] + }, + { + "path": "binding/form_mapping_test.go", + "tokens": [ + "binding", + "form" + ] + } + ] + } + }, + { + "case_id": "gin-c3d5a28ed6d3", + "repository": "gin", + "commit": "c3d5a28ed6d3849da820195b6774d212bcc038a9", + "base_revision": "acc55e049e33b401e810dbd8c0d6dcb6b3ba2b05", + "authored_at": "2025-11-07T12:01:19+08:00", + "query": "close os.File in RunFd to prevent resource leak (#4422)", + "query_tokens": [ + "close", + "file", + "leak", + "prevent", + "resource" + ], + "specific_tokens": [ + "close", + "file", + "leak", + "prevent", + "resource" + ], + "required_files": [ + "gin.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-c677ccc40a60", + "repository": "gin", + "commit": "c677ccc40a60386565dd0d755efacb85d153feca", + "base_revision": "7e298066baab19316aa2ffc946f1bbc44a68a607", + "authored_at": "2024-05-10T07:27:42+08:00", + "query": "invalid Go toolchain version (#3961)", + "query_tokens": [ + "invalid", + "toolchain", + "version" + ], + "specific_tokens": [ + "invalid", + "toolchain", + "version" + ], + "required_files": [ + "go.mod" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-d75fcd4c9ab2", + "repository": "gin", + "commit": "d75fcd4c9ab260e5225de590f1f0f8c0e0e12d11", + "base_revision": "8d0468f72897652485933b845253386f9147a8bf", + "authored_at": "2026-06-02T07:02:14-07:00", + "query": "panic on Hijack/CloseNotify when wrapper unsupported (#4645)", + "query_tokens": [ + "close", + "hijack", + "notify", + "panic", + "unsupported", + "wrapper" + ], + "specific_tokens": [ + "close", + "hijack", + "notify", + "panic", + "unsupported", + "wrapper" + ], + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-d9307dbcbbe7", + "repository": "gin", + "commit": "d9307dbcbbe796a64d9e0ef23452da888dd7f904", + "base_revision": "da1e108614ecbbadfa5736b1b297b16121d23b9b", + "authored_at": "2026-06-22T22:34:41+07:00", + "query": "skip chmod on pre-existing dirs in SaveUploadedFile (#4702)", + "query_tokens": [ + "chmod", + "dirs", + "existing", + "file", + "save", + "skip", + "uploaded" + ], + "specific_tokens": [ + "chmod", + "dirs", + "existing", + "file", + "save", + "skip", + "uploaded" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-e0d46ded6cb6", + "repository": "gin", + "commit": "e0d46ded6cb6974d55a255ab122d1aa6ca0cd60e", + "base_revision": "4f339e6a35b163d31b30916b37f4176d385f41bd", + "authored_at": "2024-05-18T19:48:07-07:00", + "query": "verify URL is Non-nil in initQueryCache() (#3969)", + "query_tokens": [ + "cache", + "init", + "query", + "verify" + ], + "specific_tokens": [ + "cache", + "init", + "query", + "verify" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-e737e3e267be", + "repository": "gin", + "commit": "e737e3e267beb4dc3bab16cc8be59e3902d98a94", + "base_revision": "4ccfa7c275c449990818e46759d5974a953cc1c1", + "authored_at": "2025-03-20T23:35:49+08:00", + "query": "prevent duplicate decoding and add validation in decodeToml (#4193)", + "query_tokens": [ + "decode", + "decoding", + "duplicate", + "prevent", + "toml", + "validation" + ], + "specific_tokens": [ + "decode", + "decoding", + "duplicate", + "prevent", + "toml", + "validation" + ], + "required_files": [ + "binding/toml.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/toml.go", + "tokens": [ + "toml" + ] + } + ] + } + }, + { + "case_id": "gin-ea53388e6ee4", + "repository": "gin", + "commit": "ea53388e6ee4a6a0a1647b390c56eeed780e7e56", + "base_revision": "9d11234efec1e5517b2887a6e7dfbc9c017bc52c", + "authored_at": "2024-10-26T08:28:59+08:00", + "query": "Keep panic infos consistent when wildcard type build faild (#4077)", + "query_tokens": [ + "build", + "consistent", + "faild", + "infos", + "keep", + "panic", + "type", + "wildcard" + ], + "specific_tokens": [ + "build", + "consistent", + "faild", + "infos", + "keep", + "panic", + "type", + "wildcard" + ], + "required_files": [ + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + ".github/workflows/gin.yml" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-ebe5e2a6bfdc", + "repository": "gin", + "commit": "ebe5e2a6bfdca50fd44074b470ad486392e2933f", + "base_revision": "733ee094fc4aaf016fb05820f553eeb0b81d0f1f", + "authored_at": "2025-03-18T23:13:03+08:00", + "query": "move fiximports to goimports section and replace exportloopref with copyloopvar (#4167)", + "query_tokens": [ + "copyloopvar", + "exportloopref", + "fiximports", + "goimports", + "move", + "replace", + "section" + ], + "specific_tokens": [ + "copyloopvar", + "exportloopref", + "fiximports", + "goimports", + "move", + "replace", + "section" + ], + "required_files": [ + ".golangci.yml" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-f70dd00b00bc", + "repository": "gin", + "commit": "f70dd00b00bc0a46cb18b55bfe1f918d5d29b511", + "base_revision": "ab8042e9e5370bbe0e93ea5adc6e74ae4c5df95e", + "authored_at": "2024-03-12T13:49:23+08:00", + "query": "fix unit test (#3878)", + "query_tokens": [ + "test", + "unit" + ], + "specific_tokens": [ + "test", + "unit" + ], + "required_files": [ + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context_test.go", + "tokens": [ + "test" + ] + } + ] + } + } + ], + "rejected_sample": [ + { + "commit": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9", + "subject": "docs(path): fix malformed comment in cleanPath (#4723)", + "reason": "excluded-subject" + }, + { + "commit": "00cfe5aac2604e0632ee674b69a1e652a2461923", + "subject": "chore(deps): bump the actions group across 1 directory with 4 updates (#4787)", + "reason": "excluded-subject" + }, + { + "commit": "34dac209ffb6ef85cc78c5d217bbb7ad001d68fd", + "subject": "docs: fix `BindXML` comment referencing nonexistent `binding.BindXML` (#4717)", + "reason": "excluded-subject" + }, + { + "commit": "03f3e420a3c370659359fb4deb99a474229b215a", + "subject": "update validator library to version 10.30.3 (#4707)", + "reason": "not-a-fix" + }, + { + "commit": "da1e108614ecbbadfa5736b1b297b16121d23b9b", + "subject": "test(context): use t.TempDir() for SaveUploadedFile permission test on WSL (#4709)", + "reason": "not-a-fix" + }, + { + "commit": "074b669a95fd834701359acc55371e6b377618f6", + "subject": "test(response_writer): add tests for Flush() with and without http.Flusher (#4699)", + "reason": "not-a-fix" + }, + { + "commit": "2e4d4f38962a6f15ae496d59b294f307eef95429", + "subject": "chore(deps): bump github.com/quic-go/quic-go to v0.60.0 (#4713)", + "reason": "excluded-subject" + }, + { + "commit": "8d0468f72897652485933b845253386f9147a8bf", + "subject": "chore(deps): bump golang.org/x/net to v0.55.0 (#4678)", + "reason": "excluded-subject" + }, + { + "commit": "88c42635384d52ccc28bbcc2750cc040e07e2e82", + "subject": "docs(context): align inline comments in GetPostForm example (#4675)", + "reason": "excluded-subject" + }, + { + "commit": "96ece6a14123923d00081320486042f568adef32", + "subject": "feat(context): add Scheme() with proper reverse proxy support (#4655)", + "reason": "not-a-fix" + }, + { + "commit": "c79f5d466ee780c329663d5b599d40639b33cd93", + "subject": "refactor: optimize error message concatenation in default_validator (#4685)", + "reason": "not-a-fix" + }, + { + "commit": "5f4f9643258dc2a65e684b63f12c8d543c936c67", + "subject": "chore(deps): bump the actions group across 1 directory with 2 updates (#4640)", + "reason": "excluded-subject" + }, + { + "commit": "d3ffc9985281dcf4d3bef604cce4e662b1a327a6", + "subject": "test(engine): add regression tests for HandleContext with NoRoute (#4571)", + "reason": "not-a-fix" + }, + { + "commit": "ecd26c8835574b47acdc9cb6a5dddcc6fe922140", + "subject": "docs(readme): update benchmark results with latest data (#4583)", + "reason": "excluded-subject" + }, + { + "commit": "a749e4d33c2db35835581c60604720614e570cd6", + "subject": "docs: replace AUTHORS.md with link to GitHub contributors page (#4582)", + "reason": "excluded-subject" + }, + { + "commit": "65d1c470ec1d2377623dff5f65241beeed2dfb79", + "subject": "docs(benchmarks): update benchmark report with Gin v1.12.0 results (#4581)", + "reason": "excluded-subject" + }, + { + "commit": "6d880724cc553ccc902a88f7f871fd7ea0b44790", + "subject": "refactor(test): use the built-in max/min to simplify the code (#4576)", + "reason": "not-a-fix" + }, + { + "commit": "48667a2dd1172f94c43811c2545209abc59c43f4", + "subject": "chore(deps): upgrade Go dependencies and CI action versions (#4580)", + "reason": "excluded-subject" + }, + { + "commit": "d4672219fc38bd88f32da194994af8e8d718f0d8", + "subject": "docs: reorganize doc.md TOC into thematic groups (#4579)", + "reason": "excluded-subject" + }, + { + "commit": "3e44fdc4d1636a2b1599c6688a76e13216a413dd", + "subject": "chore(deps): bump aquasecurity/trivy-action in the actions group (#4557)", + "reason": "excluded-subject" + } + ] +} diff --git a/benchmarks/context-retrieval/corpora/corpus-gin76.json b/benchmarks/context-retrieval/corpora/corpus-gin76.json new file mode 100644 index 00000000..f220468a --- /dev/null +++ b/benchmarks/context-retrieval/corpora/corpus-gin76.json @@ -0,0 +1,2812 @@ +{ + "schema_version": "codevetter.context-retrieval-corpus.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "selection": { + "commits_scanned": 600, + "fixes_only": true, + "max_required_files": 8 + }, + "counts": { + "cases": 76, + "path_leak": 23, + "baseline_blind": 53, + "multi_file": 54 + }, + "cases": [ + { + "case_id": "gin-03e5e05ae089", + "repository": "gin", + "commit": "03e5e05ae089bc989f1ca41841f05504d29e3fd9", + "base_revision": "d496f64540b6707602de50ab57aeea8ff4080b74", + "authored_at": "2021-04-09T00:27:34+08:00", + "query": "data race with trustedCIDRs (#2674) (#2675)", + "query_tokens": [ + "cidrs", + "data", + "race", + "trusted" + ], + "specific_tokens": [ + "cidrs", + "data", + "race", + "trusted" + ], + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-088cdd74d42d", + "repository": "gin", + "commit": "088cdd74d42df20d4e58aa17d3f4403a22e8a545", + "base_revision": "92dd245c9bef184fcfb5289f31f4247f9fced87b", + "authored_at": "2022-07-01T10:31:31+08:00", + "query": "Fix the value of ginSupportMinGoVer constant by semantic (#3221)", + "query_tokens": [ + "constant", + "semantic", + "support", + "value" + ], + "specific_tokens": [ + "constant", + "semantic", + "support", + "value" + ], + "required_files": [ + "debug.go", + "debug_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-09f8224593e3", + "repository": "gin", + "commit": "09f8224593e31edf3c58ab3f13bc31ef53473733", + "base_revision": "f75144a356e57c95bd21a048f0a40492dcdb33c5", + "authored_at": "2024-03-06T14:16:53+05:30", + "query": "Add fullPath in context copy (#3784)", + "query_tokens": [ + "context", + "copy", + "full", + "path" + ], + "specific_tokens": [ + "context", + "copy", + "full", + "path" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context.go", + "tokens": [ + "context" + ] + }, + { + "path": "context_test.go", + "tokens": [ + "context" + ] + } + ] + } + }, + { + "case_id": "gin-11aa11a65618", + "repository": "gin", + "commit": "11aa11a65618b6b66bf2b5123e08db6e879c4a1e", + "base_revision": "0a55865c3fc4c2b30dce86dfa9cff7654ba9bc74", + "authored_at": "2021-07-27T02:59:53+03:00", + "query": "fix readability in recovery test (#2797)", + "query_tokens": [ + "readability", + "recovery", + "test" + ], + "specific_tokens": [ + "readability", + "recovery", + "test" + ], + "required_files": [ + "recovery_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "recovery_test.go", + "tokens": [ + "recovery", + "test" + ] + } + ] + } + }, + { + "case_id": "gin-1c2aa59b20c8", + "repository": "gin", + "commit": "1c2aa59b20c8cff5b3c601708afe22100eac25e6", + "base_revision": "eb75ce0ff5d64b269f2a25e83952e374e762c4b6", + "authored_at": "2021-10-26T18:15:29+08:00", + "query": "fix the misplacement of adding slashes (#2847)", + "query_tokens": [ + "adding", + "misplacement", + "slashes" + ], + "specific_tokens": [ + "adding", + "misplacement", + "slashes" + ], + "required_files": [ + "routes_test.go", + "tree.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-1d0f938f28d7", + "repository": "gin", + "commit": "1d0f938f28d7e592623264591605dc22ac141cc5", + "base_revision": "f2bbdfe9f26d84cb994f381050692a9e4553bf75", + "authored_at": "2021-06-25T13:22:01+08:00", + "query": "Fix insufficient slice check (#2755)", + "query_tokens": [ + "check", + "insufficient", + "slice" + ], + "specific_tokens": [ + "check", + "insufficient", + "slice" + ], + "required_files": [ + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-212267d67163", + "repository": "gin", + "commit": "212267d6716324536be15512c0b6ed080794b798", + "base_revision": "b682b8a54e165088efa71d8b941e7ad28cde348b", + "authored_at": "2022-11-08T19:54:48+08:00", + "query": "fix typo in comment (#3371)", + "query_tokens": [ + "comment", + "typo" + ], + "specific_tokens": [ + "comment", + "typo" + ], + "required_files": [ + "context.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-234a6d4c00cb", + "repository": "gin", + "commit": "234a6d4c00cb77af9852aca0b8289745d5529b4b", + "base_revision": "df2753778e7bc5c2dd559361cf0c97b2b313e9bb", + "authored_at": "2025-09-26T08:13:39+08:00", + "query": "refine hijack behavior for response lifecycle (#4373)", + "query_tokens": [ + "behavior", + "hijack", + "lifecycle", + "refine", + "response" + ], + "specific_tokens": [ + "behavior", + "hijack", + "lifecycle", + "refine", + "response" + ], + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "response_writer.go", + "tokens": [ + "response" + ] + }, + { + "path": "response_writer_test.go", + "tokens": [ + "response" + ] + } + ] + } + }, + { + "case_id": "gin-24a1d2adb9db", + "repository": "gin", + "commit": "24a1d2adb9dba64d38426d76f2a4cf76123a4711", + "base_revision": "4c64f1c3859fa853659ff1db65a8f0fa67856ed9", + "authored_at": "2022-10-16T11:41:14+10:00", + "query": "spelling `covert` -> `convert` (#3325)", + "query_tokens": [ + "convert", + "covert", + "spelling" + ], + "specific_tokens": [ + "convert", + "covert", + "spelling" + ], + "required_files": [ + "binding/form_mapping.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-28e57f58b184", + "repository": "gin", + "commit": "28e57f58b184b2305ace192e02496bb89f6fd8cb", + "base_revision": "3cb30679b5e3021db16c776ed7e70d380586e9ce", + "authored_at": "2024-09-06T08:21:19+03:00", + "query": "Set default value for form fields (#4047)", + "query_tokens": [ + "default", + "fields", + "form", + "value" + ], + "specific_tokens": [ + "default", + "fields", + "form", + "value" + ], + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/form_mapping.go", + "tokens": [ + "form" + ] + }, + { + "path": "binding/form_mapping_test.go", + "tokens": [ + "form" + ] + } + ] + } + }, + { + "case_id": "gin-2921582d11d5", + "repository": "gin", + "commit": "2921582d11d517dff4cf0226bfbcd8bc7c63d536", + "base_revision": "4fe5f3e4b4fe62c057ec22caee4908beeef5f59c", + "authored_at": "2021-05-19T10:05:36+08:00", + "query": "Fix conflict between param and exact path (#2706)", + "query_tokens": [ + "conflict", + "exact", + "param", + "path" + ], + "specific_tokens": [ + "conflict", + "exact", + "param", + "path" + ], + "required_files": [ + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-293ad7edebb3", + "repository": "gin", + "commit": "293ad7edebb3ae30369288bd6416ca0d78474727", + "base_revision": "2e4d4f38962a6f15ae496d59b294f307eef95429", + "authored_at": "2026-06-22T14:39:26+03:30", + "query": "Copy() copies Errors and Accepted fields (#4695)", + "query_tokens": [ + "accepted", + "copies", + "copy", + "errors", + "fields" + ], + "specific_tokens": [ + "accepted", + "copies", + "copy", + "fields" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-2a794cd0b0fa", + "repository": "gin", + "commit": "2a794cd0b0faa7d829291375b27a3467ea972b0d", + "base_revision": "b917b14ff9d189f16a7492be79d123a47806ee19", + "authored_at": "2025-12-04T11:49:37+09:00", + "query": "version mismatch (#4403)", + "query_tokens": [ + "mismatch", + "version" + ], + "specific_tokens": [ + "mismatch", + "version" + ], + "required_files": [ + "debug.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-2ae61570499d", + "repository": "gin", + "commit": "2ae61570499d8bb5eb05e46d22a3754cf2635e63", + "base_revision": "fb13e822a4b48b872dd4538dd5bbd7a70767ca7a", + "authored_at": "2022-08-31T01:34:33-05:00", + "query": "Fix typos in RouterGroup method docs (#3302)", + "query_tokens": [ + "docs", + "group", + "method", + "router", + "typos" + ], + "specific_tokens": [ + "docs", + "group", + "method", + "router", + "typos" + ], + "required_files": [ + "routergroup.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-2b1da2b0b38d", + "repository": "gin", + "commit": "2b1da2b0b38dfc5d5841266037c0c8b249eca1dd", + "base_revision": "0d9dbbb44551a872d30fd89d4d55ba0515d646fd", + "authored_at": "2024-03-21T21:08:41+08:00", + "query": "make context Value method adhere to Go standards (#3897)", + "query_tokens": [ + "adhere", + "context", + "make", + "method", + "standards", + "value" + ], + "specific_tokens": [ + "adhere", + "context", + "make", + "method", + "standards", + "value" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context.go", + "tokens": [ + "context" + ] + }, + { + "path": "context_test.go", + "tokens": [ + "context" + ] + } + ] + } + }, + { + "case_id": "gin-2d4bbec94155", + "repository": "gin", + "commit": "2d4bbec941551479b1fdf1e54ece03e6e82a7e72", + "base_revision": "9f5ecd4be440f2789db917aa93c1b8afa2276917", + "authored_at": "2023-05-29T10:57:53+09:00", + "query": "fix lack of escaping of filename in Content-Disposition (#3556)", + "query_tokens": [ + "content", + "disposition", + "escaping", + "filename", + "lack" + ], + "specific_tokens": [ + "content", + "disposition", + "escaping", + "filename", + "lack" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-328d0b8076f0", + "repository": "gin", + "commit": "328d0b8076f08b58efbe052f9fa8b7ea0807bddd", + "base_revision": "f07a4f8aea8ba84dcfe46196f3c43aa3f4e2349e", + "authored_at": "2021-05-24T11:55:54+03:00", + "query": "Fixed typo in documentation (#2733)", + "query_tokens": [ + "documentation", + "fixed", + "typo" + ], + "specific_tokens": [ + "documentation", + "fixed", + "typo" + ], + "required_files": [ + "fs.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-36b0dede4b8c", + "repository": "gin", + "commit": "36b0dede4b8c4a67d92c4107cebc5a068364321d", + "base_revision": "3f5b0afa2ac85ea79638ca08f4140ce64b8246e5", + "authored_at": "2024-05-13T14:55:41+08:00", + "query": "check handler is nil (#3413)", + "query_tokens": [ + "check", + "handler" + ], + "specific_tokens": [ + "check", + "handler" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-386d244068db", + "repository": "gin", + "commit": "386d244068db3693f938db4ead6d1f5f85942e3f", + "base_revision": "44d0dd70924dd154e3b98bc340accc53484efa9c", + "authored_at": "2023-12-07T00:38:55Z", + "query": "correctly expand the capacity of params (#3502)", + "query_tokens": [ + "capacity", + "correctly", + "expand", + "params" + ], + "specific_tokens": [ + "capacity", + "expand", + "params" + ], + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-3a6f18f32f22", + "repository": "gin", + "commit": "3a6f18f32f22d7978bbafdf9b81d3a568b7a5868", + "base_revision": "ae349b4015f4736e44ea813365dcf51094329b95", + "authored_at": "2021-09-08T11:30:55+08:00", + "query": "fixed SetOutput() panics on go 1.17 (#2861)", + "query_tokens": [ + "fixed", + "output", + "panics" + ], + "specific_tokens": [ + "fixed", + "output", + "panics" + ], + "required_files": [ + "go.mod", + "go.sum" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-3dc1cd6572b4", + "repository": "gin", + "commit": "3dc1cd6572b4e3a0cd170a15debe546c2c72294f", + "base_revision": "9f598a31aafb92d675f38f1c8371e4ac76f858bf", + "authored_at": "2024-02-05T10:46:35+08:00", + "query": "binding error while not upload file (#3819) (#3820)", + "query_tokens": [ + "binding", + "error", + "file", + "upload" + ], + "specific_tokens": [ + "binding", + "file", + "upload" + ], + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/form_mapping.go", + "tokens": [ + "binding" + ] + }, + { + "path": "binding/form_mapping_test.go", + "tokens": [ + "binding" + ] + } + ] + } + }, + { + "case_id": "gin-44d0dd70924d", + "repository": "gin", + "commit": "44d0dd70924dd154e3b98bc340accc53484efa9c", + "base_revision": "49f45a542719df661bd71dd48f1595f0bc1ff6f7", + "authored_at": "2023-11-16T21:16:43+05:30", + "query": "Add pointer support for url query params (#3659) (#3666)", + "query_tokens": [ + "params", + "pointer", + "query", + "support" + ], + "specific_tokens": [ + "params", + "pointer", + "query", + "support" + ], + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-472d086af2ac", + "repository": "gin", + "commit": "472d086af2acd924cb4b9d7be0525f7d790f69bc", + "base_revision": "fb2583442c4d9bccb75e6d26f1aa6e7c01950db6", + "authored_at": "2026-02-27T19:15:27-08:00", + "query": "panic in findCaseInsensitivePathRec with RedirectFixedPath (#4535)", + "query_tokens": [ + "case", + "find", + "fixed", + "insensitive", + "panic", + "path", + "redirect" + ], + "specific_tokens": [ + "case", + "find", + "fixed", + "insensitive", + "panic", + "path", + "redirect" + ], + "required_files": [ + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-4a3eb31fb15b", + "repository": "gin", + "commit": "4a3eb31fb15b2a2d78b4bdbe0c31a2c564b1977a", + "base_revision": "293ad7edebb3ae30369288bd6416ca0d78474727", + "authored_at": "2026-06-22T20:56:28+08:00", + "query": "record recovered panics in c.Errors (#4698)", + "query_tokens": [ + "errors", + "panics", + "record", + "recovered" + ], + "specific_tokens": [ + "panics", + "record", + "recovered" + ], + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-4a40f8f1a49b", + "repository": "gin", + "commit": "4a40f8f1a49b9086b461d97e167c3b9628d8b923", + "base_revision": "857db39f82fb82456af2906ccea972ae1d65ff57", + "authored_at": "2024-02-01T09:00:17+08:00", + "query": "upgrade golang.org/x/crypto to 0.17.0 (#3832)", + "query_tokens": [ + "crypto", + "golang", + "upgrade" + ], + "specific_tokens": [ + "crypto", + "golang", + "upgrade" + ], + "required_files": [ + "go.mod", + "go.sum" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-4cee78f5382d", + "repository": "gin", + "commit": "4cee78f5382d5245c3652e6c15fee715eec505c3", + "base_revision": "fc1c43298de675e5252d0b44f97dc5e204bd4e1e", + "authored_at": "2023-02-19T22:25:48+09:00", + "query": "Fix #3500 Add escape logic for header (#3503)", + "query_tokens": [ + "escape", + "header", + "logic" + ], + "specific_tokens": [ + "escape", + "header", + "logic" + ], + "required_files": [ + "gin.go", + "routes_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-4f339e6a35b1", + "repository": "gin", + "commit": "4f339e6a35b163d31b30916b37f4176d385f41bd", + "base_revision": "36b0dede4b8c4a67d92c4107cebc5a068364321d", + "authored_at": "2024-05-14T10:25:54+08:00", + "query": "YAML judgment logic in Negotiate (#3966)", + "query_tokens": [ + "judgment", + "logic", + "negotiate", + "yaml" + ], + "specific_tokens": [ + "judgment", + "logic", + "negotiate", + "yaml" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-51aea73ba0f1", + "repository": "gin", + "commit": "51aea73ba0f125f6cacc3b4b695efdf21d9c634f", + "base_revision": "33ab0fc155e68acbc7fd30281c0d9b2e6e091b7b", + "authored_at": "2022-10-20T00:49:19+08:00", + "query": "modify interface check way (#3327)", + "query_tokens": [ + "check", + "interface", + "modify" + ], + "specific_tokens": [ + "check", + "interface", + "modify" + ], + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-55e27f12465e", + "repository": "gin", + "commit": "55e27f12465e058058180280d5f0bdc473eb3302", + "base_revision": "971fe21876e491f2597a390522adc849272e3bec", + "authored_at": "2022-11-06T17:08:11+08:00", + "query": "missing route params for CreateTestContext (#2778) (#2803)", + "query_tokens": [ + "context", + "create", + "missing", + "params", + "route", + "test" + ], + "specific_tokens": [ + "context", + "create", + "missing", + "params", + "route", + "test" + ], + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.667, + "shared_tokens": [ + { + "path": "context_test.go", + "tokens": [ + "context", + "test" + ] + }, + { + "path": "test_helpers.go", + "tokens": [ + "test" + ] + } + ] + } + }, + { + "case_id": "gin-5826722a87cf", + "repository": "gin", + "commit": "5826722a87cf5855fcc4b794cbef11612352771d", + "base_revision": "bdc1ad7987b6931801af9d50e2df25667fdfaaf4", + "authored_at": "2025-07-17T20:51:40+09:00", + "query": "version number discrepancy (#4299)", + "query_tokens": [ + "discrepancy", + "number", + "version" + ], + "specific_tokens": [ + "discrepancy", + "number", + "version" + ], + "required_files": [ + "debug.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-5c00df8afadd", + "repository": "gin", + "commit": "5c00df8afadd06cc5be530dde00fe6d9fa4a2e4a", + "base_revision": "db309081bc5c137b2aa15701ef53f7f19788da25", + "authored_at": "2026-02-28T09:07:31+07:00", + "query": "write content length in Data.Render (#4206)", + "query_tokens": [ + "content", + "data", + "length", + "render", + "write" + ], + "specific_tokens": [ + "content", + "data", + "length", + "render", + "write" + ], + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "render/data.go", + "tokens": [ + "data", + "render" + ] + }, + { + "path": "render/render_test.go", + "tokens": [ + "render" + ] + } + ] + } + }, + { + "case_id": "gin-5fad976b372e", + "repository": "gin", + "commit": "5fad976b372e381312f8de69f0969f1284d229d3", + "base_revision": "93ff771e6dbf10e432864b30f3719ac5c84a4d4a", + "authored_at": "2025-11-16T06:52:07+05:30", + "query": "literal colon routes not working with engine.Handler() (#4415)", + "query_tokens": [ + "colon", + "engine", + "handler", + "literal", + "routes", + "working" + ], + "specific_tokens": [ + "colon", + "engine", + "handler", + "literal", + "routes", + "working" + ], + "required_files": [ + "gin.go", + "gin_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-626d55b0c029", + "repository": "gin", + "commit": "626d55b0c02937645c21774cacc021713de88604", + "base_revision": "9c081de9cdd1948f521d47d170d18cbc2981c33a", + "authored_at": "2024-06-22T16:19:04+02:00", + "query": "Do not panic when handling method not allowed on empty tree (#4003)", + "query_tokens": [ + "allowed", + "empty", + "handling", + "method", + "panic", + "tree" + ], + "specific_tokens": [ + "allowed", + "empty", + "method", + "panic", + "tree" + ], + "required_files": [ + "gin.go", + "gin_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-63dd3e60cab8", + "repository": "gin", + "commit": "63dd3e60cab89c27fb66bce1423bd268d52abad1", + "base_revision": "c358d5656d0feb8b310d4ec379bccde46ccc8cc7", + "authored_at": "2025-11-27T23:20:52+08:00", + "query": "suppress http.ErrAbortHandler in recover (#4336)", + "query_tokens": [ + "abort", + "handler", + "http", + "recover", + "suppress" + ], + "specific_tokens": [ + "abort", + "handler", + "http", + "recover", + "suppress" + ], + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-646312aef6a3", + "repository": "gin", + "commit": "646312aef6a34095476ac846b0920db5fb24b2ea", + "base_revision": "5f458dd1a6d631f324e4af9a4f5429ffdf199342", + "authored_at": "2024-03-11T22:24:36+08:00", + "query": "protect Context.Keys map when call Copy method (#3873)", + "query_tokens": [ + "call", + "context", + "copy", + "keys", + "method", + "protect" + ], + "specific_tokens": [ + "call", + "context", + "copy", + "keys", + "method", + "protect" + ], + "required_files": [ + "context.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context.go", + "tokens": [ + "context" + ] + } + ] + } + }, + { + "case_id": "gin-674522db91d6", + "repository": "gin", + "commit": "674522db91d637d179c16c372d87756ea26fa089", + "base_revision": "8f7c340577e19245827f7ba71ef3e0143cc7eeee", + "authored_at": "2025-05-21T19:21:46+08:00", + "query": "binding time with empty value (#4103)", + "query_tokens": [ + "binding", + "empty", + "time", + "value" + ], + "specific_tokens": [ + "binding", + "empty", + "time", + "value" + ], + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/form_mapping.go", + "tokens": [ + "binding" + ] + }, + { + "path": "binding/form_mapping_test.go", + "tokens": [ + "binding" + ] + } + ] + } + }, + { + "case_id": "gin-68542126982e", + "repository": "gin", + "commit": "68542126982eb2275ea0447c58b38cb38cff8ddc", + "base_revision": "888b14ab283f3623174e99b6ae7555060731cd89", + "authored_at": "2022-04-17T12:41:59+08:00", + "query": "missing `sameSite` when do context.reset() (#3123)", + "query_tokens": [ + "context", + "missing", + "reset", + "site" + ], + "specific_tokens": [ + "context", + "missing", + "reset", + "site" + ], + "required_files": [ + "context.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context.go", + "tokens": [ + "context" + ] + } + ] + } + }, + { + "case_id": "gin-7742ff50e0a0", + "repository": "gin", + "commit": "7742ff50e0a05d079a0c468ccfbf7c6ecfe2414b", + "base_revision": "65ed60ed1334140d8583a3d0533cea76c66b69fe", + "authored_at": "2020-11-11T09:41:35+08:00", + "query": "Fix typos in context.go (#2551)", + "query_tokens": [ + "context", + "typos" + ], + "specific_tokens": [ + "context", + "typos" + ], + "required_files": [ + "context.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context.go", + "tokens": [ + "context" + ] + } + ] + } + }, + { + "case_id": "gin-7a1b655074c3", + "repository": "gin", + "commit": "7a1b655074c313f9491c83bb8ea164cdc4a9afe9", + "base_revision": "67c9d4ee110e9adfe33063ef847dba56717c148a", + "authored_at": "2025-05-11T20:04:09+05:30", + "query": "sonic on arm64 (#4234)", + "query_tokens": [ + "arm64", + "sonic" + ], + "specific_tokens": [ + "arm64", + "sonic" + ], + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + ".github/workflows/gin.yml", + "docs/doc.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.25, + "shared_tokens": [ + { + "path": "internal/json/sonic.go", + "tokens": [ + "sonic" + ] + } + ] + } + }, + { + "case_id": "gin-7cb151bb4c4c", + "repository": "gin", + "commit": "7cb151bb4c4cfc6018a00a125422ff38a041b9f8", + "base_revision": "3010cbd7f4eccdbb610c510274895e083b8c058c", + "authored_at": "2023-01-16T15:50:07+01:00", + "query": "panic on NegotiateFormat - index out of range (#3397)", + "query_tokens": [ + "format", + "index", + "negotiate", + "panic", + "range" + ], + "specific_tokens": [ + "format", + "index", + "negotiate", + "panic", + "range" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-7d147928ee23", + "repository": "gin", + "commit": "7d147928ee232fce156ea7ce8ae6329e148aeb41", + "base_revision": "f5f5da8fa09d12a22225c493fa8191fb14bdd5bf", + "authored_at": "2024-05-08T04:13:36+03:00", + "query": "data race warning for gin mode (#1580)", + "query_tokens": [ + "data", + "mode", + "race", + "warning" + ], + "specific_tokens": [ + "data", + "mode", + "race", + "warning" + ], + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.667, + "shared_tokens": [ + { + "path": "mode.go", + "tokens": [ + "mode" + ] + }, + { + "path": "mode_test.go", + "tokens": [ + "mode" + ] + } + ] + } + }, + { + "case_id": "gin-7d189814cbf6", + "repository": "gin", + "commit": "7d189814cbf6d4c55e8a47b06234736cb1e03457", + "base_revision": "e56d18f19d1f89c68de06e66272373287a4497d3", + "authored_at": "2021-12-12T13:30:33+08:00", + "query": "wrong when wildcard follows named param (#2983)", + "query_tokens": [ + "follows", + "named", + "param", + "wildcard", + "wrong" + ], + "specific_tokens": [ + "follows", + "named", + "param", + "wildcard" + ], + "required_files": [ + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-814cd188eb0b", + "repository": "gin", + "commit": "814cd188eb0b0304dd3c37b698edc51ff46e24a2", + "base_revision": "2c9e5fe47ae55defc55dae30a0e63192e4538bc2", + "authored_at": "2022-09-18T16:59:57+03:00", + "query": "FIX TYPO: Gin by default useR -> ... useS (#3324)", + "query_tokens": [ + "default", + "typo" + ], + "specific_tokens": [ + "default", + "typo" + ], + "required_files": [ + "routergroup.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-815122a0f4b9", + "repository": "gin", + "commit": "815122a0f4b95f12295921af7e04e0225eb9a6cc", + "base_revision": "05caa5c00e552a27c3bfb659b99c0d79b81fafe4", + "authored_at": "2022-06-15T17:31:44+08:00", + "query": "Fix a syntax error in a code comment (#3201)", + "query_tokens": [ + "code", + "comment", + "error", + "syntax" + ], + "specific_tokens": [ + "code", + "comment", + "syntax" + ], + "required_files": [ + "tree.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-82bcd6d39bfe", + "repository": "gin", + "commit": "82bcd6d39bfe9c22032764ff3b0b6f8ef1673e49", + "base_revision": "86ff4a64c7efe1a1c875529835eeef9e15de1e86", + "authored_at": "2024-02-07T06:44:11-05:00", + "query": "dereference pointer to struct (#3199)", + "query_tokens": [ + "dereference", + "pointer", + "struct" + ], + "specific_tokens": [ + "dereference", + "pointer", + "struct" + ], + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-86ff4a64c7ef", + "repository": "gin", + "commit": "86ff4a64c7efe1a1c875529835eeef9e15de1e86", + "base_revision": "e957d1abf13846e458956d8c97e7b7c76c7ee9a3", + "authored_at": "2024-02-06T04:08:56+01:00", + "query": "Allow header according to RFC 7231 (HTTP 405) (#3759)", + "query_tokens": [ + "according", + "allow", + "header", + "http" + ], + "specific_tokens": [ + "according", + "allow", + "header", + "http" + ], + "required_files": [ + "gin.go", + "routes_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-8763f33c65f7", + "repository": "gin", + "commit": "8763f33c65f7df8be5b9fe7504ab7fcf20abb41d", + "base_revision": "e737e3e267beb4dc3bab16cc8be59e3902d98a94", + "authored_at": "2025-03-20T17:40:41+02:00", + "query": "prevent middleware re-entry issue in HandleContext (#3987)", + "query_tokens": [ + "context", + "entry", + "handle", + "issue", + "middleware", + "prevent" + ], + "specific_tokens": [ + "context", + "entry", + "middleware", + "prevent" + ], + "required_files": [ + "gin.go", + "gin_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-87811a97bddb", + "repository": "gin", + "commit": "87811a97bddbe552ddd7f5ae2fa7bb949e787862", + "base_revision": "f1e942889abdab773538f215ec6aa242f994c6d0", + "authored_at": "2022-05-28T08:14:35+08:00", + "query": "the trusted proxies should support ipv6 address by default (#3033)", + "query_tokens": [ + "address", + "default", + "ipv6", + "proxies", + "support", + "trusted" + ], + "specific_tokens": [ + "address", + "default", + "ipv6", + "proxies", + "support", + "trusted" + ], + "required_files": [ + "gin.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-8790d08909fc", + "repository": "gin", + "commit": "8790d08909fc4d193c6c787c9c72f3089168f411", + "base_revision": "78f4687875d72d10392f8a77008cbefdec4c0aa0", + "authored_at": "2024-03-21T16:28:42+02:00", + "query": "query binding bug (#3236)", + "query_tokens": [ + "binding", + "query" + ], + "specific_tokens": [ + "binding", + "query" + ], + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/query.go", + "tokens": [ + "binding", + "query" + ] + }, + { + "path": "binding/query_test.go", + "tokens": [ + "binding", + "query" + ] + } + ] + } + }, + { + "case_id": "gin-8b9c55e8b059", + "repository": "gin", + "commit": "8b9c55e8b059a1ffcc69e3b5dc0b9d583958811f", + "base_revision": "51aea73ba0f125f6cacc3b4b695efdf21d9c634f", + "authored_at": "2022-11-06T17:02:40+08:00", + "query": "redirectSlash bug (#3227)", + "query_tokens": [ + "redirect", + "slash" + ], + "specific_tokens": [ + "redirect", + "slash" + ], + "required_files": [ + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-8dd20118ba9a", + "repository": "gin", + "commit": "8dd20118ba9a9dfa18ebba99e33d6a59b0090436", + "base_revision": "00cfe5aac2604e0632ee674b69a1e652a2461923", + "authored_at": "2026-08-15T11:13:07+05:30", + "query": "bump golang.org/x/net and golang.org/x/text to patched versions (#4807)", + "query_tokens": [ + "bump", + "golang", + "patched", + "text", + "versions" + ], + "specific_tokens": [ + "bump", + "golang", + "patched", + "text", + "versions" + ], + "required_files": [ + "go.mod", + "go.sum" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-8e07d37c63e5", + "repository": "gin", + "commit": "8e07d37c63e5536eb25f4af4c91eabeee4011fba", + "base_revision": "d7776de7d444935ea4385999711bd6331a98fecb", + "authored_at": "2026-02-13T11:39:14+05:45", + "query": "Correct typos, improve documentation clarity, and remove dead code (#4511)", + "query_tokens": [ + "clarity", + "code", + "correct", + "dead", + "documentation", + "improve", + "remove", + "typos" + ], + "specific_tokens": [ + "clarity", + "code", + "dead", + "documentation", + "remove", + "typos" + ], + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "CONTRIBUTING.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-8eb5f832bac1", + "repository": "gin", + "commit": "8eb5f832bac1853fc84c508a2b9406134d39492e", + "base_revision": "c58e0d59ca6753da47daa0a64b844bc869030759", + "authored_at": "2023-01-07T01:57:54+01:00", + "query": "tree bug where loop index is not decremented. (#3460)", + "query_tokens": [ + "decremented", + "index", + "loop", + "tree" + ], + "specific_tokens": [ + "decremented", + "index", + "loop", + "tree" + ], + "required_files": [ + "routes_test.go", + "tree.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "tree.go", + "tokens": [ + "tree" + ] + } + ] + } + }, + { + "case_id": "gin-97b3c0d88ada", + "repository": "gin", + "commit": "97b3c0d88ada639763414f35715f88d6bd38fa10", + "base_revision": "f469c1be3925ab32bd38fbf944b230c70abe9e2b", + "authored_at": "2021-09-29T00:35:06+01:00", + "query": "Fix grammatical and spelling errors in context.go (#2883)", + "query_tokens": [ + "context", + "errors", + "grammatical", + "spelling" + ], + "specific_tokens": [ + "context", + "grammatical", + "spelling" + ], + "required_files": [ + "context.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context.go", + "tokens": [ + "context" + ] + } + ] + } + }, + { + "case_id": "gin-9914178584e4", + "repository": "gin", + "commit": "9914178584e42458ff7d23891463a880f58c9d86", + "base_revision": "915e4c90d28ec4cffc6eb146e208ab5a65eac772", + "authored_at": "2026-01-02T02:15:27Z", + "query": "ClientIP handling for multiple X-Forwarded-For header values (#4472)", + "query_tokens": [ + "client", + "forwarded", + "handling", + "header", + "multiple", + "values" + ], + "specific_tokens": [ + "client", + "forwarded", + "header", + "multiple", + "values" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-9f598a31aafb", + "repository": "gin", + "commit": "9f598a31aafb92d675f38f1c8371e4ac76f858bf", + "base_revision": "c6ae2e69666a2b36203b29650ee75d172c725c66", + "authored_at": "2024-02-04T18:44:29+05:30", + "query": "catch-all conflicting wildcard (#3812)", + "query_tokens": [ + "catch", + "conflicting", + "wildcard" + ], + "specific_tokens": [ + "conflicting", + "wildcard" + ], + "required_files": [ + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-aa6002134e97", + "repository": "gin", + "commit": "aa6002134e97efefd879b6819c1bfce114b05f42", + "base_revision": "87811a97bddbe552ddd7f5ae2fa7bb949e787862", + "authored_at": "2022-05-28T02:27:10+02:00", + "query": "Fix intercepting headers in middlewares (#1271)", + "query_tokens": [ + "headers", + "intercepting", + "middlewares" + ], + "specific_tokens": [ + "headers", + "intercepting", + "middlewares" + ], + "required_files": [ + "context_go17_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-aefae309a4fc", + "repository": "gin", + "commit": "aefae309a4fc197ce5d57cd8391562b6d2a63a95", + "base_revision": "8edb7a71a17061bbaa85db53bb8ba6daad3c3aa8", + "authored_at": "2022-11-06T17:12:11+08:00", + "query": "test fmt.Println replace t.Error (#3328)", + "query_tokens": [ + "error", + "println", + "replace", + "test" + ], + "specific_tokens": [ + "println", + "replace", + "test" + ], + "required_files": [ + "gin_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "gin_test.go", + "tokens": [ + "test" + ] + } + ] + } + }, + { + "case_id": "gin-b38c59de7fef", + "repository": "gin", + "commit": "b38c59de7fef67400a1c98efeae700a689c45783", + "base_revision": "cf32d2dcf8c7534f59727c5e213e45f2412c593a", + "authored_at": "2025-05-11T18:38:33+04:00", + "query": "change Unwrap method receiver to value type (#4232)", + "query_tokens": [ + "change", + "method", + "receiver", + "type", + "unwrap", + "value" + ], + "specific_tokens": [ + "method", + "receiver", + "type", + "unwrap", + "value" + ], + "required_files": [ + "errors.go", + "errors_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-b917b14ff9d1", + "repository": "gin", + "commit": "b917b14ff9d189f16a7492be79d123a47806ee19", + "base_revision": "fad706f1216e6d12bdd51d28d5a40ec27e6c6453", + "authored_at": "2025-12-03T19:18:10+08:00", + "query": "empty value error (#2169)", + "query_tokens": [ + "empty", + "error", + "value" + ], + "specific_tokens": [ + "empty", + "value" + ], + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-bb1fc2e0fe97", + "repository": "gin", + "commit": "bb1fc2e0fe97c63dab1527baab88d01183853b8f", + "base_revision": "2d4bbec941551479b1fdf1e54ece03e6e82a7e72", + "authored_at": "2023-05-29T01:59:35Z", + "query": "fix Request.Context() checks (#3512)", + "query_tokens": [ + "checks", + "context", + "request" + ], + "specific_tokens": [ + "checks", + "context", + "request" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context.go", + "tokens": [ + "context" + ] + }, + { + "path": "context_test.go", + "tokens": [ + "context" + ] + } + ] + } + }, + { + "case_id": "gin-be860ec1578f", + "repository": "gin", + "commit": "be860ec1578f0c9d1e220934455e9507d3a4c1e6", + "base_revision": "dd8a27c0b6fbbfd192c5586ba3165700f535ac7f", + "authored_at": "2021-06-24T13:07:49+08:00", + "query": "fix typo and add comments (#2760)", + "query_tokens": [ + "comments", + "typo" + ], + "specific_tokens": [ + "comments", + "typo" + ], + "required_files": [ + "recovery.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-c3d1092b3b48", + "repository": "gin", + "commit": "c3d1092b3b48addf6f9cd00fe274ec3bd14650eb", + "base_revision": "9968c4bf9d5a99edc3eee2c068a4c9160ece8915", + "authored_at": "2025-10-11T19:20:41+08:00", + "query": "improve empty slice/array handling in form binding (#4380)", + "query_tokens": [ + "array", + "binding", + "empty", + "form", + "handling", + "improve", + "slice" + ], + "specific_tokens": [ + "array", + "binding", + "empty", + "form", + "slice" + ], + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/form_mapping.go", + "tokens": [ + "binding", + "form" + ] + }, + { + "path": "binding/form_mapping_test.go", + "tokens": [ + "binding", + "form" + ] + } + ] + } + }, + { + "case_id": "gin-c3d5a28ed6d3", + "repository": "gin", + "commit": "c3d5a28ed6d3849da820195b6774d212bcc038a9", + "base_revision": "acc55e049e33b401e810dbd8c0d6dcb6b3ba2b05", + "authored_at": "2025-11-07T12:01:19+08:00", + "query": "close os.File in RunFd to prevent resource leak (#4422)", + "query_tokens": [ + "close", + "file", + "leak", + "prevent", + "resource" + ], + "specific_tokens": [ + "close", + "file", + "leak", + "prevent", + "resource" + ], + "required_files": [ + "gin.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-c677ccc40a60", + "repository": "gin", + "commit": "c677ccc40a60386565dd0d755efacb85d153feca", + "base_revision": "7e298066baab19316aa2ffc946f1bbc44a68a607", + "authored_at": "2024-05-10T07:27:42+08:00", + "query": "invalid Go toolchain version (#3961)", + "query_tokens": [ + "invalid", + "toolchain", + "version" + ], + "specific_tokens": [ + "invalid", + "toolchain", + "version" + ], + "required_files": [ + "go.mod" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-c706ace9298f", + "repository": "gin", + "commit": "c706ace9298fe6440041ba0cb099ea4b3acda980", + "base_revision": "444e156fb1ec2943af5308a790315833c74fdc5a", + "authored_at": "2022-04-23T12:01:03+02:00", + "query": "removed YODA conditions, removed blank identifier from `invalid_obj` (#3129)", + "query_tokens": [ + "blank", + "conditions", + "identifier", + "invalid", + "removed", + "yoda" + ], + "specific_tokens": [ + "blank", + "conditions", + "identifier", + "invalid", + "removed", + "yoda" + ], + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-cbdd47a7e108", + "repository": "gin", + "commit": "cbdd47a7e108217d4672f0e24d423b632a8cd900", + "base_revision": "d4e72a17f7771d929a35bc8bb561ee24676dac1a", + "authored_at": "2021-10-31T20:21:37-04:00", + "query": "fix tsr with mixed static and wildcard paths (#2924)", + "query_tokens": [ + "mixed", + "paths", + "static", + "wildcard" + ], + "specific_tokens": [ + "mixed", + "paths", + "static", + "wildcard" + ], + "required_files": [ + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-d75fcd4c9ab2", + "repository": "gin", + "commit": "d75fcd4c9ab260e5225de590f1f0f8c0e0e12d11", + "base_revision": "8d0468f72897652485933b845253386f9147a8bf", + "authored_at": "2026-06-02T07:02:14-07:00", + "query": "panic on Hijack/CloseNotify when wrapper unsupported (#4645)", + "query_tokens": [ + "close", + "hijack", + "notify", + "panic", + "unsupported", + "wrapper" + ], + "specific_tokens": [ + "close", + "hijack", + "notify", + "panic", + "unsupported", + "wrapper" + ], + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-d9307dbcbbe7", + "repository": "gin", + "commit": "d9307dbcbbe796a64d9e0ef23452da888dd7f904", + "base_revision": "da1e108614ecbbadfa5736b1b297b16121d23b9b", + "authored_at": "2026-06-22T22:34:41+07:00", + "query": "skip chmod on pre-existing dirs in SaveUploadedFile (#4702)", + "query_tokens": [ + "chmod", + "dirs", + "existing", + "file", + "save", + "skip", + "uploaded" + ], + "specific_tokens": [ + "chmod", + "dirs", + "existing", + "file", + "save", + "skip", + "uploaded" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-e0d46ded6cb6", + "repository": "gin", + "commit": "e0d46ded6cb6974d55a255ab122d1aa6ca0cd60e", + "base_revision": "4f339e6a35b163d31b30916b37f4176d385f41bd", + "authored_at": "2024-05-18T19:48:07-07:00", + "query": "verify URL is Non-nil in initQueryCache() (#3969)", + "query_tokens": [ + "cache", + "init", + "query", + "verify" + ], + "specific_tokens": [ + "cache", + "init", + "query", + "verify" + ], + "required_files": [ + "context.go", + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-e737e3e267be", + "repository": "gin", + "commit": "e737e3e267beb4dc3bab16cc8be59e3902d98a94", + "base_revision": "4ccfa7c275c449990818e46759d5974a953cc1c1", + "authored_at": "2025-03-20T23:35:49+08:00", + "query": "prevent duplicate decoding and add validation in decodeToml (#4193)", + "query_tokens": [ + "decode", + "decoding", + "duplicate", + "prevent", + "toml", + "validation" + ], + "specific_tokens": [ + "decode", + "decoding", + "duplicate", + "prevent", + "toml", + "validation" + ], + "required_files": [ + "binding/toml.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/toml.go", + "tokens": [ + "toml" + ] + } + ] + } + }, + { + "case_id": "gin-ea53388e6ee4", + "repository": "gin", + "commit": "ea53388e6ee4a6a0a1647b390c56eeed780e7e56", + "base_revision": "9d11234efec1e5517b2887a6e7dfbc9c017bc52c", + "authored_at": "2024-10-26T08:28:59+08:00", + "query": "Keep panic infos consistent when wildcard type build faild (#4077)", + "query_tokens": [ + "build", + "consistent", + "faild", + "infos", + "keep", + "panic", + "type", + "wildcard" + ], + "specific_tokens": [ + "build", + "consistent", + "faild", + "infos", + "keep", + "panic", + "type", + "wildcard" + ], + "required_files": [ + "tree.go", + "tree_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + ".github/workflows/gin.yml" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-eab47b5423b6", + "repository": "gin", + "commit": "eab47b5423b67608ffe099ce2f851f3d48ef4a96", + "base_revision": "deb83b6365cf1b95f50929492f6058c94a6ff274", + "authored_at": "2021-09-07T10:10:32+08:00", + "query": "check obj type in protobufBinding (#2851)", + "query_tokens": [ + "binding", + "check", + "protobuf", + "type" + ], + "specific_tokens": [ + "binding", + "check", + "protobuf", + "type" + ], + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "binding/binding_test.go", + "tokens": [ + "binding" + ] + }, + { + "path": "binding/protobuf.go", + "tokens": [ + "binding", + "protobuf" + ] + } + ] + } + }, + { + "case_id": "gin-ebe5e2a6bfdc", + "repository": "gin", + "commit": "ebe5e2a6bfdca50fd44074b470ad486392e2933f", + "base_revision": "733ee094fc4aaf016fb05820f553eeb0b81d0f1f", + "authored_at": "2025-03-18T23:13:03+08:00", + "query": "move fiximports to goimports section and replace exportloopref with copyloopvar (#4167)", + "query_tokens": [ + "copyloopvar", + "exportloopref", + "fiximports", + "goimports", + "move", + "replace", + "section" + ], + "specific_tokens": [ + "copyloopvar", + "exportloopref", + "fiximports", + "goimports", + "move", + "replace", + "section" + ], + "required_files": [ + ".golangci.yml" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-f4bc259de33c", + "repository": "gin", + "commit": "f4bc259de33c561fd3b0ae3e7aaa849c1d251c0b", + "base_revision": "e753c502dcbbab6769305871d700c770e68d1b0f", + "authored_at": "2021-01-12T08:32:04+08:00", + "query": "fix error gin support min Go version (#2584)", + "query_tokens": [ + "error", + "support", + "version" + ], + "specific_tokens": [ + "support", + "version" + ], + "required_files": [ + "debug.go", + "debug_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "gin-f70dd00b00bc", + "repository": "gin", + "commit": "f70dd00b00bc0a46cb18b55bfe1f918d5d29b511", + "base_revision": "ab8042e9e5370bbe0e93ea5adc6e74ae4c5df95e", + "authored_at": "2024-03-12T13:49:23+08:00", + "query": "fix unit test (#3878)", + "query_tokens": [ + "test", + "unit" + ], + "specific_tokens": [ + "test", + "unit" + ], + "required_files": [ + "context_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "context_test.go", + "tokens": [ + "test" + ] + } + ] + } + } + ], + "rejected_sample": [ + { + "commit": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9", + "subject": "docs(path): fix malformed comment in cleanPath (#4723)", + "reason": "excluded-subject" + }, + { + "commit": "00cfe5aac2604e0632ee674b69a1e652a2461923", + "subject": "chore(deps): bump the actions group across 1 directory with 4 updates (#4787)", + "reason": "excluded-subject" + }, + { + "commit": "34dac209ffb6ef85cc78c5d217bbb7ad001d68fd", + "subject": "docs: fix `BindXML` comment referencing nonexistent `binding.BindXML` (#4717)", + "reason": "excluded-subject" + }, + { + "commit": "03f3e420a3c370659359fb4deb99a474229b215a", + "subject": "update validator library to version 10.30.3 (#4707)", + "reason": "not-a-fix" + }, + { + "commit": "da1e108614ecbbadfa5736b1b297b16121d23b9b", + "subject": "test(context): use t.TempDir() for SaveUploadedFile permission test on WSL (#4709)", + "reason": "not-a-fix" + }, + { + "commit": "074b669a95fd834701359acc55371e6b377618f6", + "subject": "test(response_writer): add tests for Flush() with and without http.Flusher (#4699)", + "reason": "not-a-fix" + }, + { + "commit": "2e4d4f38962a6f15ae496d59b294f307eef95429", + "subject": "chore(deps): bump github.com/quic-go/quic-go to v0.60.0 (#4713)", + "reason": "excluded-subject" + }, + { + "commit": "8d0468f72897652485933b845253386f9147a8bf", + "subject": "chore(deps): bump golang.org/x/net to v0.55.0 (#4678)", + "reason": "excluded-subject" + }, + { + "commit": "88c42635384d52ccc28bbcc2750cc040e07e2e82", + "subject": "docs(context): align inline comments in GetPostForm example (#4675)", + "reason": "excluded-subject" + }, + { + "commit": "96ece6a14123923d00081320486042f568adef32", + "subject": "feat(context): add Scheme() with proper reverse proxy support (#4655)", + "reason": "not-a-fix" + }, + { + "commit": "c79f5d466ee780c329663d5b599d40639b33cd93", + "subject": "refactor: optimize error message concatenation in default_validator (#4685)", + "reason": "not-a-fix" + }, + { + "commit": "5f4f9643258dc2a65e684b63f12c8d543c936c67", + "subject": "chore(deps): bump the actions group across 1 directory with 2 updates (#4640)", + "reason": "excluded-subject" + }, + { + "commit": "d3ffc9985281dcf4d3bef604cce4e662b1a327a6", + "subject": "test(engine): add regression tests for HandleContext with NoRoute (#4571)", + "reason": "not-a-fix" + }, + { + "commit": "ecd26c8835574b47acdc9cb6a5dddcc6fe922140", + "subject": "docs(readme): update benchmark results with latest data (#4583)", + "reason": "excluded-subject" + }, + { + "commit": "a749e4d33c2db35835581c60604720614e570cd6", + "subject": "docs: replace AUTHORS.md with link to GitHub contributors page (#4582)", + "reason": "excluded-subject" + }, + { + "commit": "65d1c470ec1d2377623dff5f65241beeed2dfb79", + "subject": "docs(benchmarks): update benchmark report with Gin v1.12.0 results (#4581)", + "reason": "excluded-subject" + }, + { + "commit": "6d880724cc553ccc902a88f7f871fd7ea0b44790", + "subject": "refactor(test): use the built-in max/min to simplify the code (#4576)", + "reason": "not-a-fix" + }, + { + "commit": "48667a2dd1172f94c43811c2545209abc59c43f4", + "subject": "chore(deps): upgrade Go dependencies and CI action versions (#4580)", + "reason": "excluded-subject" + }, + { + "commit": "d4672219fc38bd88f32da194994af8e8d718f0d8", + "subject": "docs: reorganize doc.md TOC into thematic groups (#4579)", + "reason": "excluded-subject" + }, + { + "commit": "3e44fdc4d1636a2b1599c6688a76e13216a413dd", + "subject": "chore(deps): bump aquasecurity/trivy-action in the actions group (#4557)", + "reason": "excluded-subject" + } + ] +} diff --git a/benchmarks/context-retrieval/corpora/corpus-got.json b/benchmarks/context-retrieval/corpora/corpus-got.json new file mode 100644 index 00000000..e3caea6a --- /dev/null +++ b/benchmarks/context-retrieval/corpora/corpus-got.json @@ -0,0 +1,4098 @@ +{ + "schema_version": "codevetter.context-retrieval-corpus.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "selection": { + "commits_scanned": 600, + "fixes_only": true, + "max_required_files": 8 + }, + "counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cases": [ + { + "case_id": "got-043c9501b851", + "repository": "got", + "commit": "043c9501b85172e09819d44ac8eb49c574b27bda", + "base_revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "authored_at": "2020-07-28T10:59:11+02:00", + "query": "Fix `dnsCache: true` having no effect", + "query_tokens": [ + "cache", + "effect", + "having", + "true" + ], + "specific_tokens": [ + "cache", + "effect", + "having", + "true" + ], + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-05cb01405a70", + "repository": "got", + "commit": "05cb01405a7043701916a82eec1f021603e62df7", + "base_revision": "6a544a392bafd66c4a1d4ec0643d0d4203d70a79", + "authored_at": "2025-10-11T11:55:10+09:00", + "query": "Fix outdated 'How to store options' docs", + "query_tokens": [ + "docs", + "options", + "outdated", + "store" + ], + "specific_tokens": [ + "docs", + "options", + "outdated", + "store" + ], + "required_files": [ + "test/arguments.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "documentation/2-options.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-06029e0b758e", + "repository": "got", + "commit": "06029e0b758e99f4369cc7382fae907adc9f83b1", + "base_revision": "383a3ff4ff88d4059519efe8b3d6b27e9a6a8066", + "authored_at": "2025-03-31T13:59:18+01:00", + "query": "Gracefully handle undefined request (#2404)", + "query_tokens": [ + "gracefully", + "handle", + "request", + "undefined" + ], + "specific_tokens": [ + "gracefully", + "request", + "undefined" + ], + "required_files": [ + "source/core/index.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-06a2d3d7d8d4", + "repository": "got", + "commit": "06a2d3d7d8d4fcc6898b6364d1a18ca1d407092b", + "base_revision": "6e1d9f97465d79980b03ed69518ae73a26e12c75", + "authored_at": "2021-06-03T01:37:38+02:00", + "query": "Fix a Node.js 16 bug that hangs Got streams", + "query_tokens": [ + "hangs", + "node", + "streams" + ], + "specific_tokens": [ + "hangs", + "node", + "streams" + ], + "required_files": [ + "source/core/index.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-0703318d3bb6", + "repository": "got", + "commit": "0703318d3bb6cc1340614612d2ae14b23f651242", + "base_revision": "dbbd31777b3299ceb0db99798453e01ab8e597b1", + "authored_at": "2022-03-02T02:26:39+01:00", + "query": "Fix encoding with json `responseType` (#1996)", + "query_tokens": [ + "encoding", + "json", + "response", + "type" + ], + "specific_tokens": [ + "encoding", + "json", + "response", + "type" + ], + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "source/core/response.ts", + "tokens": [ + "response" + ] + }, + { + "path": "test/encoding.ts", + "tokens": [ + "encoding" + ] + } + ] + } + }, + { + "case_id": "got-071ea0749d81", + "repository": "got", + "commit": "071ea0749d813f34d8458dbc1fb5e3dc66e80890", + "base_revision": "b03e6f95ebb5fe3c51512fe0190a20385e2f9ee9", + "authored_at": "2026-04-20T23:13:31+07:00", + "query": "Fix false `ReadError` on responses without `Content-Length`", + "query_tokens": [ + "content", + "error", + "false", + "length", + "read", + "responses" + ], + "specific_tokens": [ + "content", + "false", + "length", + "read", + "responses" + ], + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-0863bcd5e90e", + "repository": "got", + "commit": "0863bcd5e90e029518c7d9f16dbd0a8761112e9a", + "base_revision": "4e7567965dbc0e66bf73b9e53be8b33cda4b3842", + "authored_at": "2025-10-10T21:22:58+09:00", + "query": "Fix flaky 'timeouts are emitted ASAP' test", + "query_tokens": [ + "asap", + "emitted", + "flaky", + "test", + "timeouts" + ], + "specific_tokens": [ + "asap", + "emitted", + "flaky", + "test", + "timeouts" + ], + "required_files": [ + "test/timeout.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/timeout.ts", + "tokens": [ + "test" + ] + } + ] + } + }, + { + "case_id": "got-08d7a0f3285d", + "repository": "got", + "commit": "08d7a0f3285d5ac535aac873cd9090937cf008b6", + "base_revision": "0fb6ec60d299fd9b48966608a4c3f201746d821c", + "authored_at": "2021-08-21T23:52:25+02:00", + "query": "Fix assertions when cloning raw options", + "query_tokens": [ + "assertions", + "cloning", + "options" + ], + "specific_tokens": [ + "assertions", + "cloning", + "options" + ], + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "source/core/options.ts", + "tokens": [ + "options" + ] + } + ] + } + }, + { + "case_id": "got-0a16a9ca7de3", + "repository": "got", + "commit": "0a16a9ca7de3a3571c2a9f617af1cf94f09e17c8", + "base_revision": "ff46eaecba4303dc71034552c70c31718b0e9807", + "authored_at": "2025-11-02T16:23:56+07:00", + "query": "Fix path segments containing colons being misidentified as absolute URLs", + "query_tokens": [ + "absolute", + "colons", + "containing", + "misidentified", + "path", + "segments", + "urls" + ], + "specific_tokens": [ + "absolute", + "colons", + "containing", + "misidentified", + "path", + "segments", + "urls" + ], + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-1018c2029eea", + "repository": "got", + "commit": "1018c2029eea1f5b75b5120265996f1c0b3c12ae", + "base_revision": "25101f228572ebeeb90c74a0e6433b389f7d3fff", + "authored_at": "2021-07-30T14:09:21+02:00", + "query": "Fix merging `searchParams`", + "query_tokens": [ + "merging", + "params", + "search" + ], + "specific_tokens": [ + "merging", + "params", + "search" + ], + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "documentation/2-options.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-11a2202de328", + "repository": "got", + "commit": "11a2202de328abc668cee5593ec59077c103d88e", + "base_revision": "e9489c1729a56711a18aa49d11c1ed596dfef451", + "authored_at": "2026-05-06T21:15:37+09:00", + "query": "Fix aborting during download progress", + "query_tokens": [ + "aborting", + "download", + "progress" + ], + "specific_tokens": [ + "aborting", + "download", + "progress" + ], + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-15395ee87e6d", + "repository": "got", + "commit": "15395ee87e6daad41ed06fa8cd6cb2ad11150e6b", + "base_revision": "fbf9e693a63d2a141ff5cd51f94a27edbc5c1250", + "authored_at": "2021-06-02T00:20:49+02:00", + "query": "Fix a typo in a test name (#1738)", + "query_tokens": [ + "name", + "test", + "typo" + ], + "specific_tokens": [ + "name", + "test", + "typo" + ], + "required_files": [ + "test/normalize-arguments.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/normalize-arguments.ts", + "tokens": [ + "test" + ] + } + ] + } + }, + { + "case_id": "got-157e02bcb613", + "repository": "got", + "commit": "157e02bcb613ab506829326e7af96e8155b4e186", + "base_revision": "7b19e8f32c90984a168ac2e29585bcd637535e9a", + "authored_at": "2020-05-03T15:41:08+02:00", + "query": "Fix `got.mergeOptions()` regression", + "query_tokens": [ + "merge", + "options", + "regression" + ], + "specific_tokens": [ + "merge", + "options", + "regression" + ], + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "readme.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-1b208df66716", + "repository": "got", + "commit": "1b208df667169249d4392214b7771546ceede204", + "base_revision": "e1c18444715b4e326a14aa2d90c71967c74b08ac", + "authored_at": "2020-09-18T16:08:25+02:00", + "query": "Fix a stream test", + "query_tokens": [ + "stream", + "test" + ], + "specific_tokens": [ + "stream", + "test" + ], + "required_files": [ + "test/stream.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/stream.ts", + "tokens": [ + "stream", + "test" + ] + } + ] + } + }, + { + "case_id": "got-1e1e50647e93", + "repository": "got", + "commit": "1e1e50647e93d038a4cc6a9bbbfbf61165d8fd39", + "base_revision": "7524d9051d96c10924045d902ae219857ca3e57b", + "authored_at": "2021-07-01T14:37:28+09:30", + "query": "Fix default pagination handling for empty `Link` header (#1768)", + "query_tokens": [ + "default", + "empty", + "handling", + "header", + "link", + "pagination" + ], + "specific_tokens": [ + "default", + "empty", + "header", + "link", + "pagination" + ], + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/pagination.ts", + "tokens": [ + "pagination" + ] + } + ] + } + }, + { + "case_id": "got-1e49781eff61", + "repository": "got", + "commit": "1e49781eff61a568af68a0f328bb245b2effe9cb", + "base_revision": "91cdc48d4b07586e4e46ed3846749e345511facc", + "authored_at": "2025-10-13T14:50:24+09:00", + "query": "Fix race condition causing retry after promise settles", + "query_tokens": [ + "causing", + "condition", + "promise", + "race", + "retry", + "settles" + ], + "specific_tokens": [ + "causing", + "condition", + "promise", + "race", + "retry", + "settles" + ], + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "source/as-promise/index.ts", + "tokens": [ + "promise" + ] + }, + { + "path": "test/retry.ts", + "tokens": [ + "retry" + ] + } + ] + } + }, + { + "case_id": "got-226cc3995f6e", + "repository": "got", + "commit": "226cc3995f6e16938163ebde24d8762e7dcd15e2", + "base_revision": "c0b5346433996e761bca15cdb942f2b740264118", + "authored_at": "2021-07-12T16:35:12+02:00", + "query": "Fix `Cannot call end` error when `request` returns a `Writable`", + "query_tokens": [ + "call", + "cannot", + "error", + "request", + "returns", + "writable" + ], + "specific_tokens": [ + "call", + "cannot", + "request", + "returns", + "writable" + ], + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-2453e5e4213f", + "repository": "got", + "commit": "2453e5e4213fe036a0108de3e4db414dcf2b4c30", + "base_revision": "846e298b6b48a7b78e46eb487b4fdc0ffc0f8cd9", + "authored_at": "2021-07-31T21:59:31+02:00", + "query": "Fix unhandled exception when lookup returns invalid IP early", + "query_tokens": [ + "early", + "exception", + "invalid", + "lookup", + "returns", + "unhandled" + ], + "specific_tokens": [ + "early", + "exception", + "invalid", + "lookup", + "returns", + "unhandled" + ], + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-2527bf690fd5", + "repository": "got", + "commit": "2527bf690fd528a8480a8c5cbe5edef73a190167", + "base_revision": "30b3b795580d7d2a62e85382af5483bcc9a60899", + "authored_at": "2025-10-12T02:02:43+09:00", + "query": "Fix stream validation errors causing unhandled rejections", + "query_tokens": [ + "causing", + "errors", + "rejections", + "stream", + "unhandled", + "validation" + ], + "specific_tokens": [ + "causing", + "rejections", + "stream", + "unhandled", + "validation" + ], + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/stream.ts", + "tokens": [ + "stream" + ] + } + ] + } + }, + { + "case_id": "got-26fffe94618b", + "repository": "got", + "commit": "26fffe94618b370629f6148b6d21ebfeab1e5299", + "base_revision": "b3fd50c7f6b1686106c2cae1c65b1b3d2ed9df30", + "authored_at": "2022-09-12T20:45:48+02:00", + "query": "Correct RFC reference in TS docs as well (#2144)", + "query_tokens": [ + "correct", + "docs", + "reference", + "well" + ], + "specific_tokens": [ + "docs", + "reference", + "well" + ], + "required_files": [ + "source/core/options.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-2ab94fd4daf6", + "repository": "got", + "commit": "2ab94fd4daf6e6194ea79b810172b4e9466d3e39", + "base_revision": "f1cc7b28404c20c536030281f99725e06c362bc3", + "authored_at": "2025-10-10T10:49:46+09:00", + "query": "Fix hang on revalidated cached responses", + "query_tokens": [ + "cached", + "hang", + "responses", + "revalidated" + ], + "specific_tokens": [ + "cached", + "hang", + "responses", + "revalidated" + ], + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "derivable_files": [], + "incidental_files": [ + "package.json" + ], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-2ccc4c2e8bd5", + "repository": "got", + "commit": "2ccc4c2e8bd5fb32a647f13627eaed279625f8f2", + "base_revision": "2c8fe196361e5fc3c063b56b816f916999d60c09", + "authored_at": "2020-05-31T21:00:51+02:00", + "query": "Fix hanging promise on aborted requests on Node v14.3.0 (#1296)", + "query_tokens": [ + "aborted", + "hanging", + "node", + "promise", + "requests" + ], + "specific_tokens": [ + "aborted", + "hanging", + "node", + "promise", + "requests" + ], + "required_files": [ + "source/core/index.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-30b3b795580d", + "repository": "got", + "commit": "30b3b795580d7d2a62e85382af5483bcc9a60899", + "base_revision": "6ae3e7fc364b9a34ed6e276ab4671be43f660840", + "authored_at": "2025-10-12T00:31:50+09:00", + "query": "Fix incorrect `content-length` when piping decompressed responses", + "query_tokens": [ + "content", + "decompressed", + "incorrect", + "length", + "piping", + "responses" + ], + "specific_tokens": [ + "content", + "decompressed", + "incorrect", + "length", + "piping", + "responses" + ], + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "documentation/3-streams.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-32070619a547", + "repository": "got", + "commit": "32070619a5479419acd209ed8cea8d4bbc39b00f", + "base_revision": "461b3d41790c7f091ebbb9acd87d25a22e0ffb1a", + "authored_at": "2022-07-24T01:17:43+02:00", + "query": "Fix `ciphers` test", + "query_tokens": [ + "ciphers", + "test" + ], + "specific_tokens": [ + "ciphers", + "test" + ], + "required_files": [ + "test/https.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/https.ts", + "tokens": [ + "test" + ] + } + ] + } + }, + { + "case_id": "got-398c11ae502c", + "repository": "got", + "commit": "398c11ae502c032cb91883df3bb1d9f51271e7e4", + "base_revision": "1c3a041685f4bf077ee8ddd0c8a1ca4d145d3c60", + "authored_at": "2025-10-15T20:19:47+09:00", + "query": "Fix HTTP/2 timings NaN issue", + "query_tokens": [ + "http", + "issue", + "timings" + ], + "specific_tokens": [ + "http", + "timings" + ], + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/timings.ts", + "tokens": [ + "timings" + ] + } + ] + } + }, + { + "case_id": "got-3b3ea6776b4f", + "repository": "got", + "commit": "3b3ea6776b4f15d69d1b4dde87f32173a3a6fdb2", + "base_revision": "3e9d3af606c38f60e6225f8e009aacd6962fef71", + "authored_at": "2022-09-27T14:17:17+07:00", + "query": "Fix compatibility with TypeScript and ESM", + "query_tokens": [ + "compatibility", + "script", + "type" + ], + "specific_tokens": [ + "compatibility", + "script", + "type" + ], + "required_files": [ + "package.json", + "tsconfig.json" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-3cc40b549c4a", + "repository": "got", + "commit": "3cc40b549c4af3617e8f6793d07533f7d4123f90", + "base_revision": "5f278d74125608b7abe75941cb6a71e21e0fb892", + "authored_at": "2022-11-16T22:15:54+13:00", + "query": "Fix abort event listeners not always being cleaned up (#2162)", + "query_tokens": [ + "abort", + "cleaned", + "event", + "listeners" + ], + "specific_tokens": [ + "abort", + "cleaned", + "event", + "listeners" + ], + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/abort.ts", + "tokens": [ + "abort" + ] + } + ] + } + }, + { + "case_id": "got-3d66aecacd6d", + "repository": "got", + "commit": "3d66aecacd6dbc94c420d25fae63f01f7f4af06f", + "base_revision": "0863bcd5e90e029518c7d9f16dbd0a8761112e9a", + "authored_at": "2025-10-10T21:56:28+09:00", + "query": "Fix DNS timing being non-zero when connecting to IP addresses", + "query_tokens": [ + "addresses", + "connecting", + "timing", + "zero" + ], + "specific_tokens": [ + "addresses", + "connecting", + "timing", + "zero" + ], + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-3e9d3af606c3", + "repository": "got", + "commit": "3e9d3af606c38f60e6225f8e009aacd6962fef71", + "base_revision": "6c7ebab3674fbf8a21275cfe9e06f71740219c05", + "authored_at": "2022-09-27T08:00:29+02:00", + "query": "Fix request body not being properly cached (#2150)", + "query_tokens": [ + "body", + "cached", + "properly", + "request" + ], + "specific_tokens": [ + "body", + "cached", + "request" + ], + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-408e22aed993", + "repository": "got", + "commit": "408e22aed99300afd4af49a4eb50963665d8a144", + "base_revision": "d12d6af280356b8e8fe5d3ae52efbfc065caae9d", + "authored_at": "2020-09-05T17:00:56+02:00", + "query": "Fix `options.port` on redirect (#1439)", + "query_tokens": [ + "options", + "port", + "redirect" + ], + "specific_tokens": [ + "options", + "port", + "redirect" + ], + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-439fb82d2a07", + "repository": "got", + "commit": "439fb82d2a07cece417a18c47e37cfdeaaf38db7", + "base_revision": "6a44a8158a0f62ded055625f9aeec028abc880f3", + "authored_at": "2021-08-02T09:02:19+02:00", + "query": "Fix relative URLs when paginating", + "query_tokens": [ + "paginating", + "relative", + "urls" + ], + "specific_tokens": [ + "paginating", + "relative", + "urls" + ], + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "documentation/4-pagination.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-462bc6300150", + "repository": "got", + "commit": "462bc630015064fa4ad4358cf28d24f95e1c958b", + "base_revision": "1120370e05fd8d9e768677d8474d0c82cf91a6a6", + "authored_at": "2021-02-26T21:37:47+03:00", + "query": "Fix `url` not being reused on retry in rare case (#1487)", + "query_tokens": [ + "case", + "rare", + "retry", + "reused" + ], + "specific_tokens": [ + "case", + "rare", + "retry", + "reused" + ], + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-47c1afedc77b", + "repository": "got", + "commit": "47c1afedc77b27a2eb3edd85064d8532da835045", + "base_revision": "88622701c70434ddc1040c46ad16de0c0acd9143", + "authored_at": "2020-05-10T14:39:43+02:00", + "query": "Fix an invalid pagination test", + "query_tokens": [ + "invalid", + "pagination", + "test" + ], + "specific_tokens": [ + "invalid", + "pagination", + "test" + ], + "required_files": [ + "test/pagination.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "readme.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/pagination.ts", + "tokens": [ + "pagination", + "test" + ] + } + ] + } + }, + { + "case_id": "got-47c3155e4bdf", + "repository": "got", + "commit": "47c3155e4bdfcb12c32c6a7d2869ff162726b550", + "base_revision": "6633001dbdd5f8900e248a36669919e8a5eb58c6", + "authored_at": "2025-11-14T01:45:49+07:00", + "query": "Fix `dnsLookup` option type to accept Node.js `dns.lookup`", + "query_tokens": [ + "accept", + "lookup", + "node", + "option", + "type" + ], + "specific_tokens": [ + "accept", + "lookup", + "node", + "option", + "type" + ], + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-4cce4de19ed5", + "repository": "got", + "commit": "4cce4de19ed577f481ba45a023b395a29cadce86", + "base_revision": "462bc630015064fa4ad4358cf28d24f95e1c958b", + "authored_at": "2021-03-06T19:34:58+07:00", + "query": "Fix docs being out of sync from #1645", + "query_tokens": [ + "docs", + "sync" + ], + "specific_tokens": [ + "docs", + "sync" + ], + "required_files": [ + "source/as-promise/types.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "readme.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-4d5168cc7c58", + "repository": "got", + "commit": "4d5168cc7c58d14d3165f60eed0e4e37eba7965e", + "base_revision": "388df48648e7c06a17237fa817ec08cef540d7da", + "authored_at": "2025-12-29T23:47:38+01:00", + "query": "Fix stream auto-end for empty PATCH/DELETE/OPTIONS", + "query_tokens": [ + "auto", + "delete", + "empty", + "options", + "patch", + "stream" + ], + "specific_tokens": [ + "auto", + "delete", + "empty", + "options", + "patch", + "stream" + ], + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "documentation/3-streams.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "test/stream.ts", + "tokens": [ + "stream" + ] + } + ] + } + }, + { + "case_id": "got-4e7567965dbc", + "repository": "got", + "commit": "4e7567965dbc0e66bf73b9e53be8b33cda4b3842", + "base_revision": "fb8641856a133a7e80790d106b08b4b585079f20", + "authored_at": "2025-10-10T21:02:48+09:00", + "query": "Fix `timings.end` being undefined when stream is destroyed before completion", + "query_tokens": [ + "completion", + "destroyed", + "stream", + "timings", + "undefined" + ], + "specific_tokens": [ + "completion", + "destroyed", + "stream", + "timings", + "undefined" + ], + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/timings.ts", + "tokens": [ + "timings" + ] + } + ] + } + }, + { + "case_id": "got-5028c11abd21", + "repository": "got", + "commit": "5028c11abd21c9d13bb3b12abb02961ec0465a8a", + "base_revision": "7dcd1456a68a2603d30cbb1a4e453c4560bd51b2", + "authored_at": "2020-07-03T14:30:24+02:00", + "query": "Fix promise not returning Buffer on compressed response", + "query_tokens": [ + "buffer", + "compressed", + "promise", + "response", + "returning" + ], + "specific_tokens": [ + "buffer", + "compressed", + "promise", + "response", + "returning" + ], + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "source/as-promise/index.ts", + "tokens": [ + "promise" + ] + }, + { + "path": "test/promise.ts", + "tokens": [ + "promise" + ] + } + ] + } + }, + { + "case_id": "got-50ef99a9856c", + "repository": "got", + "commit": "50ef99a9856ccd37a75b42f94d023a491728a92d", + "base_revision": "e23b7f9b975cd1d70254847b5348685af315163b", + "authored_at": "2020-05-03T19:25:46+02:00", + "query": "Fixes to be compatible with Node.js 14.1.0", + "query_tokens": [ + "compatible", + "fixes", + "node" + ], + "specific_tokens": [ + "compatible", + "node" + ], + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "derivable_files": [], + "incidental_files": [ + "package.json" + ], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-5772bf24dc7d", + "repository": "got", + "commit": "5772bf24dc7dd69db52fc26f8274039f52063fd2", + "base_revision": "a5b76bffb33d5fa8b0d1393cce410b88e7c2b848", + "authored_at": "2026-06-22T00:09:51+07:00", + "query": "Fix `searchParams` setter dropping the value when a URL is set (#2454)", + "query_tokens": [ + "dropping", + "params", + "search", + "setter", + "value" + ], + "specific_tokens": [ + "dropping", + "params", + "search", + "setter", + "value" + ], + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-59ae3b062f53", + "repository": "got", + "commit": "59ae3b062f53974599fd5e674e1ce72bc68b0163", + "base_revision": "2197052de61544e00daa81fd72811afcce2a397a", + "authored_at": "2021-07-17T18:35:27+02:00", + "query": "Fix an assertion for `options.method`", + "query_tokens": [ + "assertion", + "method", + "options" + ], + "specific_tokens": [ + "assertion", + "method", + "options" + ], + "required_files": [ + "source/core/options.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "source/core/options.ts", + "tokens": [ + "options" + ] + } + ] + } + }, + { + "case_id": "got-60a441939af3", + "repository": "got", + "commit": "60a441939af36143d1ead87b79ba706fe0a5b812", + "base_revision": "7ec171475656249c90c0246f9c29e9cee83fc8ad", + "authored_at": "2025-10-11T09:20:21+09:00", + "query": "Fix `afterResponse` hook validation to allow `null` body values", + "query_tokens": [ + "allow", + "body", + "hook", + "null", + "response", + "validation", + "values" + ], + "specific_tokens": [ + "allow", + "body", + "hook", + "null", + "response", + "validation", + "values" + ], + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-61d5e3b0ebc5", + "repository": "got", + "commit": "61d5e3b0ebc58da1299f11c20abcdc75ef77b389", + "base_revision": "fb03d84ebe055a4b2e78c14b0f92f3df8b966a00", + "authored_at": "2025-11-08T00:30:42+07:00", + "query": "Fix DNS cache timing confusion", + "query_tokens": [ + "cache", + "confusion", + "timing" + ], + "specific_tokens": [ + "cache", + "confusion", + "timing" + ], + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-61d6f610ffa6", + "repository": "got", + "commit": "61d6f610ffa6b760d466ff910a8e959778ee36e0", + "base_revision": "c83393933c2b308a312ed4dc85d852c79445e400", + "authored_at": "2020-07-14T18:16:29+02:00", + "query": "Fix typings for Node.js <=12", + "query_tokens": [ + "node", + "typings" + ], + "specific_tokens": [ + "node", + "typings" + ], + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-676be6d85481", + "repository": "got", + "commit": "676be6d854818c891b5210ca7abe644f83ef8bfd", + "base_revision": "779062a1b99a9a1c13737b9fd613e185d97b158b", + "authored_at": "2020-07-05T22:29:19+02:00", + "query": "Fix TypeScript types for Promise API (#1344)", + "query_tokens": [ + "promise", + "script", + "type", + "types" + ], + "specific_tokens": [ + "promise", + "script", + "type", + "types" + ], + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.667, + "shared_tokens": [ + { + "path": "source/as-promise/core.ts", + "tokens": [ + "promise" + ] + }, + { + "path": "source/as-promise/types.ts", + "tokens": [ + "promise", + "types" + ] + } + ] + } + }, + { + "case_id": "got-67d5039ff5a5", + "repository": "got", + "commit": "67d5039ff5a5518489529f9706199234e9b64ad6", + "base_revision": "469a45554ff29ca13050a2714eabaaec9f7160f7", + "authored_at": "2023-05-27T04:05:22-03:00", + "query": "Fix `get-stream` import statement (#2266)", + "query_tokens": [ + "import", + "statement", + "stream" + ], + "specific_tokens": [ + "import", + "statement", + "stream" + ], + "required_files": [ + "source/core/index.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-6961284172c4", + "repository": "got", + "commit": "6961284172c4d5cca65ea7567abda7680a61fc14", + "base_revision": "d1d4ed2f6242c2ebfccdbfe278a582bf5ed51b8b", + "authored_at": "2025-10-09T14:12:15+09:00", + "query": "Fix validation to accept `false` as agent value", + "query_tokens": [ + "accept", + "agent", + "false", + "validation", + "value" + ], + "specific_tokens": [ + "accept", + "agent", + "false", + "validation", + "value" + ], + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/agent.ts", + "tokens": [ + "agent" + ] + } + ] + } + }, + { + "case_id": "got-6a544a392baf", + "repository": "got", + "commit": "6a544a392bafd66c4a1d4ec0643d0d4203d70a79", + "base_revision": "60a441939af36143d1ead87b79ba706fe0a5b812", + "authored_at": "2025-10-11T09:34:54+09:00", + "query": "Fix hook type definitions to reflect normalized runtime state", + "query_tokens": [ + "definitions", + "hook", + "normalized", + "reflect", + "runtime", + "state", + "type" + ], + "specific_tokens": [ + "definitions", + "hook", + "normalized", + "reflect", + "runtime", + "state", + "type" + ], + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-6ae3e7fc364b", + "repository": "got", + "commit": "6ae3e7fc364b9a34ed6e276ab4671be43f660840", + "base_revision": "e09a9bd1ca9747f3b53666dc0bca06e1d742899f", + "authored_at": "2025-10-11T19:36:19+09:00", + "query": "Fix EPIPE errors bypassing retry logic in Promise API", + "query_tokens": [ + "bypassing", + "epipe", + "errors", + "logic", + "promise", + "retry" + ], + "specific_tokens": [ + "bypassing", + "epipe", + "logic", + "promise", + "retry" + ], + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.667, + "shared_tokens": [ + { + "path": "source/as-promise/index.ts", + "tokens": [ + "promise" + ] + }, + { + "path": "test/retry.ts", + "tokens": [ + "retry" + ] + } + ] + } + }, + { + "case_id": "got-6e1d9f97465d", + "repository": "got", + "commit": "6e1d9f97465d79980b03ed69518ae73a26e12c75", + "base_revision": "a06e7585ab4503e1dd7178d83dd4443ff6dda6c1", + "authored_at": "2021-06-03T01:20:26+02:00", + "query": "Fix `Invalid URL` checks on Node.js 16", + "query_tokens": [ + "checks", + "invalid", + "node" + ], + "specific_tokens": [ + "checks", + "invalid", + "node" + ], + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.2, + "shared_tokens": [ + { + "path": "test/helpers/invalid-url.ts", + "tokens": [ + "invalid" + ] + } + ] + } + }, + { + "case_id": "got-724d5923816b", + "repository": "got", + "commit": "724d5923816b6fa06762b7e95a64cc29e642f375", + "base_revision": "dc4f1e323dbdc0c4c2dbf1e03c66b714bfb51d39", + "authored_at": "2025-10-08T16:04:36+09:00", + "query": "Fix it not using HTTP/2 connection reuse by default", + "query_tokens": [ + "connection", + "default", + "http", + "reuse", + "using" + ], + "specific_tokens": [ + "connection", + "default", + "http", + "reuse", + "using" + ], + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-728aef989c1a", + "repository": "got", + "commit": "728aef989c1a6650d2b42a7f69b603075b5d35bf", + "base_revision": "04f3ea4b7995e1609555dd13f00641b8f470de4c", + "authored_at": "2020-07-03T15:40:36+02:00", + "query": "Fix unhandled `The server aborted pending request` rejection", + "query_tokens": [ + "aborted", + "pending", + "rejection", + "request", + "server", + "unhandled" + ], + "specific_tokens": [ + "aborted", + "pending", + "rejection", + "request", + "server", + "unhandled" + ], + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-74e3167b705b", + "repository": "got", + "commit": "74e3167b705b713f00f32f48f09dce94faa20dbb", + "base_revision": "f6a058a7d1fdd0b65bb75db9faf94490fb7a66ec", + "authored_at": "2026-05-07T00:01:00+09:00", + "query": "Handle abort signals added by handlers", + "query_tokens": [ + "abort", + "added", + "handle", + "handlers", + "signals" + ], + "specific_tokens": [ + "abort", + "added", + "handlers", + "signals" + ], + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/abort.ts", + "tokens": [ + "abort" + ] + } + ] + } + }, + { + "case_id": "got-78b105b5ebcf", + "repository": "got", + "commit": "78b105b5ebcf9d79555bd80a02c8f47f94c933d8", + "base_revision": "7ccc0b8ea29de26177968a6a1e37ed71c9c9e975", + "authored_at": "2021-04-14T00:43:45+02:00", + "query": "Fix request destroy check for Node.js 12", + "query_tokens": [ + "check", + "destroy", + "node", + "request" + ], + "specific_tokens": [ + "check", + "destroy", + "node", + "request" + ], + "required_files": [ + "source/core/index.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-7a920647efa2", + "repository": "got", + "commit": "7a920647efa236bd1f80dad087a27371e5070f7b", + "base_revision": "f997bcb385d4b35ff581333e6bc9360d5ee10e71", + "authored_at": "2025-02-10T08:45:39+01:00", + "query": "Fix timeout error firing even though it's cancelled (#2399)", + "query_tokens": [ + "cancelled", + "error", + "even", + "firing", + "though", + "timeout" + ], + "specific_tokens": [ + "cancelled", + "even", + "firing", + "though", + "timeout" + ], + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "test/timeout.ts", + "tokens": [ + "timeout" + ] + } + ] + } + }, + { + "case_id": "got-7acd3801ec7d", + "repository": "got", + "commit": "7acd3801ec7dc80dabc41073af1cf2874b1867a1", + "base_revision": "6aa86f2494194907f6c6c8b4774dfa1f69df6876", + "authored_at": "2020-10-20T16:27:07+02:00", + "query": "Fix for sending files with size `0` on `stat` (#1488)", + "query_tokens": [ + "files", + "sending", + "size", + "stat" + ], + "specific_tokens": [ + "files", + "sending", + "size", + "stat" + ], + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "source/core/utils/get-body-size.ts", + "tokens": [ + "size" + ] + } + ] + } + }, + { + "case_id": "got-7b19e8f32c90", + "repository": "got", + "commit": "7b19e8f32c90984a168ac2e29585bcd637535e9a", + "base_revision": "633e46e8cb184e6a2670ffae5f8184fcf710f31b", + "authored_at": "2020-05-02T16:10:42+02:00", + "query": "Fix hanging promise when using cache", + "query_tokens": [ + "cache", + "hanging", + "promise", + "using" + ], + "specific_tokens": [ + "cache", + "hanging", + "promise", + "using" + ], + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/cache.ts", + "tokens": [ + "cache" + ] + } + ] + } + }, + { + "case_id": "got-7e1de4240dad", + "repository": "got", + "commit": "7e1de4240dad5e065a7c8993b65c18efa239b616", + "base_revision": "a59fac415ac013a48b1d514837628a5cf81d6878", + "authored_at": "2021-04-11T22:44:27+02:00", + "query": "Fix a test on Node.js 14", + "query_tokens": [ + "node", + "test" + ], + "specific_tokens": [ + "node", + "test" + ], + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/timeout.ts", + "tokens": [ + "test" + ] + } + ] + } + }, + { + "case_id": "got-88622701c704", + "repository": "got", + "commit": "88622701c70434ddc1040c46ad16de0c0acd9143", + "base_revision": "7d7361cc3845421f24288fc39464a8b38bd75d56", + "authored_at": "2020-05-10T14:23:14+02:00", + "query": "Fix reusing options when paginating", + "query_tokens": [ + "options", + "paginating", + "reusing" + ], + "specific_tokens": [ + "options", + "paginating", + "reusing" + ], + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-88b32eade80f", + "repository": "got", + "commit": "88b32eade80f964c33af2bdf3970779490a7271b", + "base_revision": "25ae938e441ee1fb35bcfa76582d1075cc5c9eef", + "authored_at": "2020-09-18T23:03:29+02:00", + "query": "Fix a regression where body was sent after redirect", + "query_tokens": [ + "body", + "redirect", + "regression", + "sent" + ], + "specific_tokens": [ + "body", + "redirect", + "regression", + "sent" + ], + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-8d3412abb8aa", + "repository": "got", + "commit": "8d3412abb8aae3cefc75dd6f9e698988ed463dd6", + "base_revision": "d325d353a28e6038140aa5b18f2b6f9f796f10b6", + "authored_at": "2020-07-03T23:55:16+02:00", + "query": "Fix `prefixUrl` not working when the `url` argument is empty", + "query_tokens": [ + "argument", + "empty", + "prefix", + "working" + ], + "specific_tokens": [ + "argument", + "empty", + "prefix", + "working" + ], + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-8f775c71c41a", + "repository": "got", + "commit": "8f775c71c41a377186c6b38c6789542bfd523039", + "base_revision": "4dbada94e261ad890d2478c313ec3d74761b659e", + "authored_at": "2020-07-04T02:22:59+02:00", + "query": "Fix non-enumerable options [such as body] not being used", + "query_tokens": [ + "body", + "enumerable", + "options", + "used" + ], + "specific_tokens": [ + "body", + "enumerable", + "options", + "used" + ], + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-91aa0ac9e676", + "repository": "got", + "commit": "91aa0ac9e676130cd4cfc1183d16208c6167b31d", + "base_revision": "7dbb9ee2bd58e8ee67ebfe734e0c62e765759593", + "authored_at": "2020-05-10T13:07:41+01:00", + "query": "Fix duplicated searchParams for pagination API (#1229)", + "query_tokens": [ + "duplicated", + "pagination", + "params", + "search" + ], + "specific_tokens": [ + "duplicated", + "pagination", + "params", + "search" + ], + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "readme.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/pagination.ts", + "tokens": [ + "pagination" + ] + } + ] + } + }, + { + "case_id": "got-92b378e5fd95", + "repository": "got", + "commit": "92b378e5fd95e891e849c7f3fa7483575ab052c3", + "base_revision": "f4f3ba88e4db2270792385e7d32a694d7432d232", + "authored_at": "2024-11-04T12:02:00+01:00", + "query": "Fix support for `AbortSignal#timeout()` (#2388)", + "query_tokens": [ + "abort", + "signal", + "support", + "timeout" + ], + "specific_tokens": [ + "abort", + "signal", + "support", + "timeout" + ], + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/abort.ts", + "tokens": [ + "abort" + ] + } + ] + } + }, + { + "case_id": "got-934211f951bf", + "repository": "got", + "commit": "934211f951bfd56751e90ec4eabe6379d8541a05", + "base_revision": "7582d916db72f7bab2b2c162d37cfcfde08bbba7", + "authored_at": "2020-07-03T13:13:11+02:00", + "query": "Fix hanging promise on timeout on HTTP error", + "query_tokens": [ + "error", + "hanging", + "http", + "promise", + "timeout" + ], + "specific_tokens": [ + "hanging", + "http", + "promise", + "timeout" + ], + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "source/as-promise/index.ts", + "tokens": [ + "promise" + ] + }, + { + "path": "test/error.ts", + "tokens": [ + "error" + ] + } + ] + } + }, + { + "case_id": "got-9725fbd109a6", + "repository": "got", + "commit": "9725fbd109a6a65788bfe49da873a850a21e9bc8", + "base_revision": "449833a227fc3b8d1a43c3618f09217d47ac55ba", + "authored_at": "2025-10-14T03:34:47+09:00", + "query": "FIx preserving `prefixUrl` in hooks", + "query_tokens": [ + "hooks", + "prefix", + "preserving" + ], + "specific_tokens": [ + "hooks", + "prefix", + "preserving" + ], + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "test/hooks.ts", + "tokens": [ + "hooks" + ] + } + ] + } + }, + { + "case_id": "got-98ca6181a039", + "repository": "got", + "commit": "98ca6181a0390450b1e29c277d4670634695ce85", + "base_revision": "bb8eca924c338ca12d5b90d6a26aa28dbddb42ee", + "authored_at": "2022-04-24T22:04:02+02:00", + "query": "Fix accidental test deletion", + "query_tokens": [ + "accidental", + "deletion", + "test" + ], + "specific_tokens": [ + "accidental", + "deletion", + "test" + ], + "required_files": [ + "test/progress.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/progress.ts", + "tokens": [ + "test" + ] + } + ] + } + }, + { + "case_id": "got-99e8d134bd9a", + "repository": "got", + "commit": "99e8d134bd9ad9453a69c8c31e1a640cc7f6f1cd", + "base_revision": "ff83e9ee739bf9064a7e574b57dc076be86422a6", + "authored_at": "2021-07-31T22:20:51+02:00", + "query": "Fix a test on Node.js < 16", + "query_tokens": [ + "node", + "test" + ], + "specific_tokens": [ + "node", + "test" + ], + "required_files": [ + "test/http.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/http.ts", + "tokens": [ + "test" + ] + } + ] + } + }, + { + "case_id": "got-9a309bdbe7e2", + "repository": "got", + "commit": "9a309bdbe7e2552c5bffbea253d58c39d6e6c3e3", + "base_revision": "a748343363ecb0347897264fb49df4a8f4793997", + "authored_at": "2020-08-06T04:13:37+02:00", + "query": "Fixed deprecationWarning on https options (#1391)", + "query_tokens": [ + "deprecation", + "fixed", + "https", + "options", + "warning" + ], + "specific_tokens": [ + "deprecation", + "fixed", + "https", + "options", + "warning" + ], + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/https.ts", + "tokens": [ + "https" + ] + } + ] + } + }, + { + "case_id": "got-9ab4cf991fd5", + "repository": "got", + "commit": "9ab4cf991fd598e326f9750cf30d2e3407683ae0", + "base_revision": "89b7fdfd4e7ea4e76258f50b70ae8a1d2aea8125", + "authored_at": "2024-09-28T05:40:44+08:00", + "query": "Fix 404 links related to cacheable-request (#2379)", + "query_tokens": [ + "cacheable", + "links", + "related", + "request" + ], + "specific_tokens": [ + "cacheable", + "links", + "related", + "request" + ], + "required_files": [ + "source/core/response.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "documentation/cache.md" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-9e15d887da3b", + "repository": "got", + "commit": "9e15d887da3b065940bbc8ca38f9c748a0bbc75e", + "base_revision": "8d6a6807883323a83ea5f126ca56b55d9ce5f299", + "authored_at": "2021-07-02T02:03:11+02:00", + "query": "Fix incorrect response.complete when using cache", + "query_tokens": [ + "cache", + "complete", + "incorrect", + "response", + "using" + ], + "specific_tokens": [ + "cache", + "complete", + "incorrect", + "response", + "using" + ], + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/cache.ts", + "tokens": [ + "cache" + ] + } + ] + } + }, + { + "case_id": "got-9ec6ff02a189", + "repository": "got", + "commit": "9ec6ff02a1890bc37b579eec4d62aa3aa564c9e7", + "base_revision": "1abeba4d112edb2313be8389e6416e650ebf4429", + "authored_at": "2025-10-10T00:54:19+09:00", + "query": "Fix `downloadProgress` firing for redirect responses", + "query_tokens": [ + "download", + "firing", + "progress", + "redirect", + "responses" + ], + "specific_tokens": [ + "download", + "firing", + "progress", + "redirect", + "responses" + ], + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-a2812de3baa0", + "repository": "got", + "commit": "a2812de3baa0ae925c2f506479ddf1607a77bdd8", + "base_revision": "9ec6ff02a1890bc37b579eec4d62aa3aa564c9e7", + "authored_at": "2025-10-10T01:09:12+09:00", + "query": "Fix handling of FormData getLength errors", + "query_tokens": [ + "data", + "errors", + "form", + "handling", + "length" + ], + "specific_tokens": [ + "data", + "form", + "length" + ], + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-a59fac415ac0", + "repository": "got", + "commit": "a59fac415ac013a48b1d514837628a5cf81d6878", + "base_revision": "45b89f5b4d2f7169586750722a0bb3a0ecfc8328", + "authored_at": "2021-04-11T22:12:12+02:00", + "query": "Fix hanging promise on HTTP/2 timeout", + "query_tokens": [ + "hanging", + "http", + "promise", + "timeout" + ], + "specific_tokens": [ + "hanging", + "http", + "promise", + "timeout" + ], + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "derivable_files": [], + "incidental_files": [ + "package.json" + ], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "test/timeout.ts", + "tokens": [ + "timeout" + ] + } + ] + } + }, + { + "case_id": "got-ac5f67dd4b92", + "repository": "got", + "commit": "ac5f67dd4b92a95db3c554f117dd2b2798e5e832", + "base_revision": "da4769e988801e9e6a1c9707af7a0d98fd14b183", + "authored_at": "2020-07-04T16:12:21+02:00", + "query": "Fix cache not working with HTTP2", + "query_tokens": [ + "cache", + "http2", + "working" + ], + "specific_tokens": [ + "cache", + "http2", + "working" + ], + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/cache.ts", + "tokens": [ + "cache" + ] + } + ] + } + }, + { + "case_id": "got-aee9249444c4", + "repository": "got", + "commit": "aee9249444c4bd01c4fd094e46b4bb5a691382f5", + "base_revision": "28c0ca3c65716cc62c0d609a7c7043df9cf01b39", + "authored_at": "2026-06-28T11:45:28+02:00", + "query": "Preserve request body on cross-origin 307 and 308 redirects (#2460)", + "query_tokens": [ + "body", + "cross", + "origin", + "preserve", + "redirects", + "request" + ], + "specific_tokens": [ + "body", + "cross", + "origin", + "preserve", + "redirects", + "request" + ], + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "documentation/2-options.md" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.667, + "shared_tokens": [ + { + "path": "test/early-307-redirects.ts", + "tokens": [ + "redirects" + ] + }, + { + "path": "test/redirects.ts", + "tokens": [ + "redirects" + ] + } + ] + } + }, + { + "case_id": "got-af928f6d3974", + "repository": "got", + "commit": "af928f6d3974d7106718d37acc6bd1d33d2cec70", + "base_revision": "702ed352ac5d8527deaf4a5821d0274afc56e9d8", + "authored_at": "2023-04-10T02:46:20-03:00", + "query": "Fix type error on build (#2251)", + "query_tokens": [ + "build", + "error", + "type" + ], + "specific_tokens": [ + "build", + "type" + ], + "required_files": [ + "source/core/index.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-b17012597e4d", + "repository": "got", + "commit": "b17012597e4d41a6fad774656f7d8858b3375dd0", + "base_revision": "c5ce415cb8fd929d542c618ecbd0e9087561bb56", + "authored_at": "2026-04-13T21:51:08+07:00", + "query": "Fix stream cookie jar completion race", + "query_tokens": [ + "completion", + "cookie", + "race", + "stream" + ], + "specific_tokens": [ + "completion", + "cookie", + "race", + "stream" + ], + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/stream.ts", + "tokens": [ + "stream" + ] + } + ] + } + }, + { + "case_id": "got-b671480715db", + "repository": "got", + "commit": "b671480715dbbff908e9a385f5e714570c663cd7", + "base_revision": "231f55d6de3be5d580cd7fcbc4cb28ff2125cade", + "authored_at": "2022-09-02T18:46:07+02:00", + "query": "Fix `options.context` being not extensible", + "query_tokens": [ + "context", + "extensible", + "options" + ], + "specific_tokens": [ + "context", + "extensible", + "options" + ], + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "source/core/options.ts", + "tokens": [ + "options" + ] + } + ] + } + }, + { + "case_id": "got-bf39d2c1c58f", + "repository": "got", + "commit": "bf39d2c1c58fb79fbe4ef47d92016dcb4f57b6bd", + "base_revision": "236e74431cb5cffd47b0ce088de1a0277a438339", + "authored_at": "2022-01-09T22:28:49+08:00", + "query": "Fix `nock` compatibility (#1959)", + "query_tokens": [ + "compatibility", + "nock" + ], + "specific_tokens": [ + "compatibility", + "nock" + ], + "required_files": [ + "source/core/options.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-bf84d3697537", + "repository": "got", + "commit": "bf84d3697537d6fb6bd120591e8660cf1d93f3bc", + "base_revision": "9725fbd109a6a65788bfe49da873a850a21e9bc8", + "authored_at": "2025-10-14T15:06:17+09:00", + "query": "Fix body reassignment in `beforeRetry` hooks", + "query_tokens": [ + "body", + "hooks", + "reassignment", + "retry" + ], + "specific_tokens": [ + "body", + "hooks", + "reassignment", + "retry" + ], + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/hooks.ts", + "tokens": [ + "hooks" + ] + } + ] + } + }, + { + "case_id": "got-c079b9366ff2", + "repository": "got", + "commit": "c079b9366ff2231a806c9746f0dfb4a0ec34e5d1", + "base_revision": "3034c2fdcebdff94907a6e015a8b154e851fc343", + "authored_at": "2024-11-23T07:43:37-08:00", + "query": "Fix a crash with Got accessing a non-existent request (#2391)", + "query_tokens": [ + "accessing", + "crash", + "existent", + "request" + ], + "specific_tokens": [ + "accessing", + "existent", + "request" + ], + "required_files": [ + "source/core/index.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-c09b9857d1b4", + "repository": "got", + "commit": "c09b9857d1b44d090b92691eef2893bc5d998bba", + "base_revision": "d24e319f8a1db89f2ab2d3abd688a56e7e1b3b8b", + "authored_at": "2022-08-29T13:26:25+02:00", + "query": "Fix tough-cookie test", + "query_tokens": [ + "cookie", + "test", + "tough" + ], + "specific_tokens": [ + "cookie", + "test", + "tough" + ], + "required_files": [ + "test/cookies.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/cookies.ts", + "tokens": [ + "test" + ] + } + ] + } + }, + { + "case_id": "got-c81a61159473", + "repository": "got", + "commit": "c81a61159473b8c12c22e00b42cac2ecdfc2ee87", + "base_revision": "897f3857a8af1baa440d5fbf3918be7bac6938bb", + "authored_at": "2024-03-07T10:47:06+03:00", + "query": "Fix error handling when UTF8 decoding fails (#2336)", + "query_tokens": [ + "decoding", + "error", + "fails", + "handling", + "utf8" + ], + "specific_tokens": [ + "decoding", + "utf8" + ], + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/error.ts", + "tokens": [ + "error" + ] + } + ] + } + }, + { + "case_id": "got-c97ce7cbde86", + "repository": "got", + "commit": "c97ce7cbde86f7cc40bd646dfc660811050a8376", + "base_revision": "1b208df667169249d4392214b7771546ceede204", + "authored_at": "2020-09-18T16:55:59+02:00", + "query": "Fix destructure error on `promsie.json()`", + "query_tokens": [ + "destructure", + "error", + "json", + "promsie" + ], + "specific_tokens": [ + "destructure", + "json", + "promsie" + ], + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-cbc890292f40", + "repository": "got", + "commit": "cbc890292f406fcbde0b48663d46b510d842694c", + "base_revision": "15c30ee79f4cac774533f2e43c5883c8a2a89ee3", + "authored_at": "2021-08-12T09:05:44+02:00", + "query": "Fix invalid `afterResponse` return check", + "query_tokens": [ + "check", + "invalid", + "response", + "return" + ], + "specific_tokens": [ + "check", + "invalid", + "response", + "return" + ], + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-cc434bc52111", + "repository": "got", + "commit": "cc434bc5211145467e962a13a8e27eef5140d6e6", + "base_revision": "972fb25b5470ebe57afb3a3a0070362f19aa2dc1", + "authored_at": "2025-09-17T05:59:47+07:00", + "query": "Fix hang with responses containing `content-encoding` headers but no body", + "query_tokens": [ + "body", + "containing", + "content", + "encoding", + "hang", + "headers", + "responses" + ], + "specific_tokens": [ + "body", + "containing", + "content", + "encoding", + "hang", + "headers", + "responses" + ], + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/no-body-status-codes.ts", + "tokens": [ + "body" + ] + } + ] + } + }, + { + "case_id": "got-d1d4ed2f6242", + "repository": "got", + "commit": "d1d4ed2f6242c2ebfccdbfe278a582bf5ed51b8b", + "base_revision": "2d8182de7175dd1dcf649026e8887a7274aa11a0", + "authored_at": "2025-10-09T14:01:35+09:00", + "query": "Fix HTTP/2 memory leak from timeout listeners with connection reuse", + "query_tokens": [ + "connection", + "http", + "leak", + "listeners", + "memory", + "reuse", + "timeout" + ], + "specific_tokens": [ + "connection", + "http", + "leak", + "listeners", + "memory", + "reuse", + "timeout" + ], + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/timeout.ts", + "tokens": [ + "timeout" + ] + } + ] + } + }, + { + "case_id": "got-d65d0caf627e", + "repository": "got", + "commit": "d65d0caf627e8d1f5367db34d7d9b55d332c1efb", + "base_revision": "eda69ff924a621e499d31cbc590993a32ddb48d3", + "authored_at": "2021-08-04T23:20:59+02:00", + "query": "Fix `username` and `password` encoding in URL", + "query_tokens": [ + "encoding", + "password", + "username" + ], + "specific_tokens": [ + "encoding", + "password", + "username" + ], + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-d8c00cfa388b", + "repository": "got", + "commit": "d8c00cfa388b0a98290035940e63e8e7c1557a5e", + "base_revision": "b9ba18a9ae8b945466e1bedeba7093648f7b2e1a", + "authored_at": "2020-05-31T21:18:59+02:00", + "query": "Fix overriding some options in a `beforeRequest` hook (#1293)", + "query_tokens": [ + "hook", + "options", + "overriding", + "request" + ], + "specific_tokens": [ + "hook", + "options", + "overriding", + "request" + ], + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-da4769e98880", + "repository": "got", + "commit": "da4769e988801e9e6a1c9707af7a0d98fd14b183", + "base_revision": "aeb2e070674a8c0567cfb6d47dbfa4a04dd48dd8", + "authored_at": "2020-07-04T15:19:13+02:00", + "query": "Fix `response` event not being emitted on cache verify request (#1305)", + "query_tokens": [ + "cache", + "emitted", + "event", + "request", + "response", + "verify" + ], + "specific_tokens": [ + "cache", + "emitted", + "event", + "request", + "response", + "verify" + ], + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/cache.ts", + "tokens": [ + "cache" + ] + } + ] + } + }, + { + "case_id": "got-dad6a91b3d11", + "repository": "got", + "commit": "dad6a91b3d114de41126af695a5789229c05d8af", + "base_revision": "b581a0a8cdeb1f44790f7e1fa62e03cec45ebe89", + "authored_at": "2025-08-30T16:11:21+07:00", + "query": "Fix infinite loop when retrying with `request.options` in `afterResponse` hook", + "query_tokens": [ + "hook", + "infinite", + "loop", + "options", + "request", + "response", + "retrying" + ], + "specific_tokens": [ + "hook", + "infinite", + "loop", + "options", + "request", + "response", + "retrying" + ], + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "source/core/options.ts", + "tokens": [ + "options" + ] + }, + { + "path": "test/infinite-loop-issue.ts", + "tokens": [ + "infinite", + "loop" + ] + } + ] + } + }, + { + "case_id": "got-dc4f1e323dbd", + "repository": "got", + "commit": "dc4f1e323dbdc0c4c2dbf1e03c66b714bfb51d39", + "base_revision": "75287d683bbc4a1389aadef567864f1115625c7d", + "authored_at": "2025-10-08T16:04:23+09:00", + "query": "Fix hang with stream requests without body for methods like OPTIONS", + "query_tokens": [ + "body", + "hang", + "like", + "methods", + "options", + "requests", + "stream" + ], + "specific_tokens": [ + "body", + "hang", + "like", + "methods", + "options", + "requests", + "stream" + ], + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/stream.ts", + "tokens": [ + "stream" + ] + } + ] + } + }, + { + "case_id": "got-dfc54d9e57b7", + "repository": "got", + "commit": "dfc54d9e57b738b4447aff08940a242ec95910ae", + "base_revision": "cc346b79561b13e3a266b22b7f612fca5a45eb1e", + "authored_at": "2024-06-06T14:26:58+02:00", + "query": "Fix missing dependency", + "query_tokens": [ + "dependency", + "missing" + ], + "specific_tokens": [ + "dependency", + "missing" + ], + "required_files": [ + "package.json" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-e02845f1aa73", + "repository": "got", + "commit": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "base_revision": "a3e171c4d43c8bbb236ef3a04feb25ba6fdf6c49", + "authored_at": "2020-07-28T10:44:16+02:00", + "query": "Fix duplicated hooks when paginating", + "query_tokens": [ + "duplicated", + "hooks", + "paginating" + ], + "specific_tokens": [ + "duplicated", + "hooks", + "paginating" + ], + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/hooks.ts", + "tokens": [ + "hooks" + ] + } + ] + } + }, + { + "case_id": "got-e09a9bd1ca97", + "repository": "got", + "commit": "e09a9bd1ca9747f3b53666dc0bca06e1d742899f", + "base_revision": "5bdd3d5f175c5e0d8e37dbb7e8a946171390de17", + "authored_at": "2025-10-11T16:56:20+09:00", + "query": "Fix silent hang when returning cached response with FormData body from beforeRequest hook", + "query_tokens": [ + "body", + "cached", + "data", + "form", + "hang", + "hook", + "request", + "response", + "returning", + "silent" + ], + "specific_tokens": [ + "body", + "cached", + "data", + "form", + "hang", + "hook", + "request", + "response", + "returning", + "silent" + ], + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-e1099fbc2aa9", + "repository": "got", + "commit": "e1099fbc2aa911cf5dfba9f5fa8a38d0aa0e6231", + "base_revision": "57367efc5ead58827f58818e6b282e7142be5673", + "authored_at": "2021-08-06T12:32:40+02:00", + "query": "Fix `https.alpnProtocols` not having an effect", + "query_tokens": [ + "alpn", + "effect", + "having", + "https", + "protocols" + ], + "specific_tokens": [ + "alpn", + "effect", + "having", + "https", + "protocols" + ], + "required_files": [ + "source/core/options.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-e4aeb8e3bfc4", + "repository": "got", + "commit": "e4aeb8e3bfc42b56a42a5e1ab32432f92676925a", + "base_revision": "4be7446baaef7ff53462b3ccfc1cf44e07d44b80", + "authored_at": "2020-07-07T14:18:41+02:00", + "query": "Fix pagination backoff test", + "query_tokens": [ + "backoff", + "pagination", + "test" + ], + "specific_tokens": [ + "backoff", + "pagination", + "test" + ], + "required_files": [ + "test/pagination.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "test/pagination.ts", + "tokens": [ + "pagination", + "test" + ] + } + ] + } + }, + { + "case_id": "got-e5659d401a4a", + "repository": "got", + "commit": "e5659d401a4a5a1108da3de79e89100c06026589", + "base_revision": "8224769a15756dd2b07954bd1526a6056f0227d2", + "authored_at": "2025-10-12T23:47:29+09:00", + "query": "Fix properly treating different UNIX socket paths as different origins", + "query_tokens": [ + "different", + "origins", + "paths", + "properly", + "socket", + "treating", + "unix" + ], + "specific_tokens": [ + "different", + "origins", + "paths", + "socket", + "treating", + "unix" + ], + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.667, + "shared_tokens": [ + { + "path": "source/core/utils/is-unix-socket-url.ts", + "tokens": [ + "socket", + "unix" + ] + }, + { + "path": "test/get-unix-socket-path.ts", + "tokens": [ + "socket", + "unix" + ] + } + ] + } + }, + { + "case_id": "got-e899c0736f6a", + "repository": "got", + "commit": "e899c0736f6a49112f98a3cebbf0fe2fac53b0c3", + "base_revision": "81fc00aa0ddc528adb9674894556d4dfd6aa624b", + "authored_at": "2025-10-09T23:09:47+09:00", + "query": "Fix TypeScript type definition for retry event's createRetryStream parameter", + "query_tokens": [ + "create", + "definition", + "event", + "parameter", + "retry", + "script", + "stream", + "type" + ], + "specific_tokens": [ + "create", + "definition", + "event", + "parameter", + "retry", + "script", + "stream", + "type" + ], + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/retry.ts", + "tokens": [ + "retry" + ] + } + ] + } + }, + { + "case_id": "got-eb045bfc324d", + "repository": "got", + "commit": "eb045bfc324de39301fc497ccaeac2f51586b102", + "base_revision": "f582a091500702c88f80526a8d1fd6e040b1eee3", + "authored_at": "2022-03-22T22:01:44+02:00", + "query": "allow other json values (#2015)", + "query_tokens": [ + "allow", + "json", + "values" + ], + "specific_tokens": [ + "allow", + "json", + "values" + ], + "required_files": [ + "source/core/options.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-ecb05343dea3", + "repository": "got", + "commit": "ecb05343dea3bd35933585a1ec5bcea01348d109", + "base_revision": "49c605ca85fa35b7c1b6819219cb7bd5209b4085", + "authored_at": "2021-12-09T19:31:03-05:00", + "query": "Fix `deflate` responses not being decompressed (#1918)", + "query_tokens": [ + "decompressed", + "deflate", + "responses" + ], + "specific_tokens": [ + "decompressed", + "deflate", + "responses" + ], + "required_files": [ + "source/as-promise/index.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-f0045640d12a", + "repository": "got", + "commit": "f0045640d12ad53eca081243ede37a5ce501cb4a", + "base_revision": "bf84d3697537d6fb6bd120591e8660cf1d93f3bc", + "authored_at": "2025-10-14T17:06:46+09:00", + "query": "Fix shortcut methods ignoring handler errors", + "query_tokens": [ + "errors", + "handler", + "ignoring", + "methods", + "shortcut" + ], + "specific_tokens": [ + "handler", + "ignoring", + "methods", + "shortcut" + ], + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-f2f8cb29fe9f", + "repository": "got", + "commit": "f2f8cb29fe9ffcae39955f9a2591cc84cec349b7", + "base_revision": "9ab4cf991fd598e326f9750cf30d2e3407683ae0", + "authored_at": "2024-10-09T13:56:20+08:00", + "query": "Fix cache with HTTP2 (#2380)", + "query_tokens": [ + "cache", + "http2" + ], + "specific_tokens": [ + "cache", + "http2" + ], + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/cache.ts", + "tokens": [ + "cache" + ] + } + ] + } + }, + { + "case_id": "got-f44ef4388e0b", + "repository": "got", + "commit": "f44ef4388e0b56c8b0abd30d5161fe39053e3ce8", + "base_revision": "bc17e8120b527bb54352474300b5c71cf465ee7f", + "authored_at": "2024-07-24T06:56:41-04:00", + "query": "Fix handling of invalid arguments (#2367)", + "query_tokens": [ + "arguments", + "handling", + "invalid" + ], + "specific_tokens": [ + "arguments", + "invalid" + ], + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "test/arguments.ts", + "tokens": [ + "arguments" + ] + } + ] + } + }, + { + "case_id": "got-f7ab6e9ffda6", + "repository": "got", + "commit": "f7ab6e9ffda6dec1821d18c070a231c9b38063ec", + "base_revision": "a149dfbf72fedc3822cc9f8157582f6b92d43774", + "authored_at": "2025-11-22T17:47:24+07:00", + "query": "Fix TypeScript type inference for `got.extend()` with `responseType` option", + "query_tokens": [ + "extend", + "inference", + "option", + "response", + "script", + "type" + ], + "specific_tokens": [ + "extend", + "inference", + "option", + "response", + "script", + "type" + ], + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.2, + "shared_tokens": [ + { + "path": "test/extend.types.ts", + "tokens": [ + "extend" + ] + } + ] + } + }, + { + "case_id": "got-fb8641856a13", + "repository": "got", + "commit": "fb8641856a133a7e80790d106b08b4b585079f20", + "base_revision": "7b32611d8e5292b5e6a073fe88e8ac812ceb9eb9", + "authored_at": "2025-10-10T18:13:46+09:00", + "query": "Fix `beforeError` hook not being called for `ERR_UNSUPPORTED_PROTOCOL` error", + "query_tokens": [ + "called", + "error", + "hook", + "protocol", + "unsupported" + ], + "specific_tokens": [ + "called", + "hook", + "protocol", + "unsupported" + ], + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "got-ff918fb6dedb", + "repository": "got", + "commit": "ff918fb6dedb6d8b23421497ec890d43f45121b7", + "base_revision": "1107cc625e4cc469276483316c48896a21f6251a", + "authored_at": "2020-12-01T03:03:07+04:00", + "query": "Fix promise shortcuts in case of error status code (#1543)", + "query_tokens": [ + "case", + "code", + "error", + "promise", + "shortcuts", + "status" + ], + "specific_tokens": [ + "case", + "code", + "promise", + "shortcuts", + "status" + ], + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "source/as-promise/index.ts", + "tokens": [ + "promise" + ] + }, + { + "path": "test/promise.ts", + "tokens": [ + "promise" + ] + } + ] + } + } + ], + "rejected_sample": [ + { + "commit": "e3924aa1e53a6ca3eb93a43618ce532442a89b40", + "subject": "Add support for `QUERY` HTTP method (#2466)", + "reason": "not-a-fix" + }, + { + "commit": "dd3b295fdc7c921d21fa046003c63cf1f6d14332", + "subject": "Various tweaks (#2465)", + "reason": "not-a-fix" + }, + { + "commit": "1e157c43c4395912d66154993875a2a6fb3dbb6e", + "subject": "Rewrite HTTP/2 support (#2464)", + "reason": "not-a-fix" + }, + { + "commit": "bfc400b2a12a0bbb4d0ed896d61d834cd5f2ebcb", + "subject": " Replace `cacheable-lookup` dependency with internal DNS cache (#2463)", + "reason": "not-a-fix" + }, + { + "commit": "b855688f0e520c82aff7a1dd913f9f95a4a05337", + "subject": "15.1.0", + "reason": "not-a-fix" + }, + { + "commit": "1c88e888352c54ab4f1b16c45b05a0060601af94", + "subject": "Add `allowAbsoluteUrls` option (#2462)", + "reason": "not-a-fix" + }, + { + "commit": "66ca4e025622e1dd71f4e4f13aa6e1b79d61b069", + "subject": "15.0.7", + "reason": "not-a-fix" + }, + { + "commit": "28c0ca3c65716cc62c0d609a7c7043df9cf01b39", + "subject": "Fix docs typos (#2458)", + "reason": "no-code-files" + }, + { + "commit": "4cc6f566eadaca4ab4a3064e94ee16072cac839b", + "subject": "15.0.6", + "reason": "not-a-fix" + }, + { + "commit": "a5b76bffb33d5fa8b0d1393cce410b88e7c2b848", + "subject": "Document pagination authorization handling", + "reason": "not-a-fix" + }, + { + "commit": "d00d845a0b435365ec73ea5a1fca6e17ee677184", + "subject": "15.0.5", + "reason": "not-a-fix" + }, + { + "commit": "f6a058a7d1fdd0b65bb75db9faf94490fb7a66ec", + "subject": "15.0.4", + "reason": "not-a-fix" + }, + { + "commit": "e9489c1729a56711a18aa49d11c1ed596dfef451", + "subject": "15.0.3", + "reason": "not-a-fix" + }, + { + "commit": "1a1452c38981d2f5f10f41392acf3dd16bb70f9b", + "subject": "Update dependencies", + "reason": "not-a-fix" + }, + { + "commit": "b03e6f95ebb5fe3c51512fe0190a20385e2f9ee9", + "subject": "Tweaks", + "reason": "not-a-fix" + }, + { + "commit": "552f0fe234745f5423e314430d9f5d77b6363ba2", + "subject": "15.0.2", + "reason": "not-a-fix" + }, + { + "commit": "c5ce415cb8fd929d542c618ecbd0e9087561bb56", + "subject": "Tweaks", + "reason": "not-a-fix" + }, + { + "commit": "aa2b3bb4ff6eada5fc766024b3b6a0c58efffb28", + "subject": "15.0.1", + "reason": "not-a-fix" + }, + { + "commit": "20633bc1d461a38d3466dd98ea0669a9d47a43aa", + "subject": "Simplify init types", + "reason": "not-a-fix" + }, + { + "commit": "4d7e628b1c288f72a28567ceabcff51bfc160db5", + "subject": "Tweaks", + "reason": "not-a-fix" + } + ] +} diff --git a/benchmarks/context-retrieval/corpora/corpus-hugo.json b/benchmarks/context-retrieval/corpora/corpus-hugo.json new file mode 100644 index 00000000..9ab50e16 --- /dev/null +++ b/benchmarks/context-retrieval/corpora/corpus-hugo.json @@ -0,0 +1,1336 @@ +{ + "schema_version": "codevetter.context-retrieval-corpus.v1", + "repository": { + "id": "hugo", + "head": "7b5199fdeff7d7682e9f979f468d55bc7b67b934" + }, + "selection": { + "commits_scanned": 600, + "fixes_only": true, + "max_required_files": 8 + }, + "counts": { + "cases": 29, + "path_leak": 13, + "baseline_blind": 16, + "multi_file": 25 + }, + "cases": [ + { + "case_id": "hugo-09048aad7655", + "repository": "hugo", + "commit": "09048aad7655385e9a287448eed65408d0b850ab", + "base_revision": "f42c422a12987c9ccb1927fc17da37feedd0c7fb", + "authored_at": "2026-01-01T13:56:55+01:00", + "query": "Fix partial decorator detection in partial with blocks with outer range break or continue", + "query_tokens": [ + "blocks", + "break", + "continue", + "decorator", + "detection", + "outer", + "partial", + "range" + ], + "specific_tokens": [ + "blocks", + "break", + "continue", + "decorator", + "detection", + "outer", + "partial", + "range" + ], + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "tpl/templates/decorator_integration_test.go", + "tokens": [ + "decorator" + ] + } + ] + } + }, + { + "case_id": "hugo-0f1c7d12000f", + "repository": "hugo", + "commit": "0f1c7d12000f7db7f1f45366c2dc4355b1511d5f", + "base_revision": "10352335e04c4779e101b6d40202dd90a170dda0", + "authored_at": "2026-02-01T22:24:52+01:00", + "query": "Fix template change detection for multi-version sites", + "query_tokens": [ + "change", + "detection", + "multi", + "sites", + "template", + "version" + ], + "specific_tokens": [ + "detection", + "multi", + "sites", + "template", + "version" + ], + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "hugolib/hugo_sites.go", + "tokens": [ + "sites" + ] + } + ] + } + }, + { + "case_id": "hugo-11f7f39913d5", + "repository": "hugo", + "commit": "11f7f39913d56291f0e049f674378da06cfd4e70", + "base_revision": "192e3c453e904cf2e79312d4335a7ee216fe659d", + "authored_at": "2026-01-23T16:08:29+01:00", + "query": "Fix cascade draft panic", + "query_tokens": [ + "cascade", + "draft", + "panic" + ], + "specific_tokens": [ + "cascade", + "draft", + "panic" + ], + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + ".github/workflows/test.yml" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "hugolib/cascade_test.go", + "tokens": [ + "cascade" + ] + } + ] + } + }, + { + "case_id": "hugo-161d0d475773", + "repository": "hugo", + "commit": "161d0d475773c5251ec00cc8848f1a174393155c", + "base_revision": "45e4596630b1372ab02634e4536d2b5a89452e0b", + "authored_at": "2026-04-07T21:02:27+02:00", + "query": "Fix RenderShortcodes leaking context markers when indented", + "query_tokens": [ + "context", + "indented", + "leaking", + "markers", + "render", + "shortcodes" + ], + "specific_tokens": [ + "context", + "indented", + "leaking", + "markers", + "render", + "shortcodes" + ], + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-2ffaf1fc1edb", + "repository": "hugo", + "commit": "2ffaf1fc1edb31993f22dd7fd8da5eeb0f4df015", + "base_revision": "a808f6e40d4cb109c65d232ec63c5433bc180aeb", + "authored_at": "2026-08-08T16:25:53+02:00", + "query": "Fix server static file detection for deleted files/directories in the static syncer", + "query_tokens": [ + "deleted", + "detection", + "directories", + "file", + "files", + "server", + "static", + "syncer" + ], + "specific_tokens": [ + "deleted", + "detection", + "directories", + "file", + "files", + "server", + "static", + "syncer" + ], + "required_files": [ + "commands/server.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "testscripts/server/server__delete_static_file_publish.txt" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "commands/server.go", + "tokens": [ + "server" + ] + } + ] + } + }, + { + "case_id": "hugo-3dff7c8c7a04", + "repository": "hugo", + "commit": "3dff7c8c7a04a413437f2f09e3a1252ae6f1be92", + "base_revision": "d98cd4aecf25b9df78d811759ea6135b0c7610f1", + "authored_at": "2026-02-25T17:35:20+01:00", + "query": "Fix menu pageRef resolution in multidimensional setups", + "query_tokens": [ + "menu", + "multidimensional", + "page", + "resolution", + "setups" + ], + "specific_tokens": [ + "menu", + "multidimensional", + "page", + "resolution", + "setups" + ], + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.5, + "shared_tokens": [ + { + "path": "hugolib/menu_test.go", + "tokens": [ + "menu" + ] + } + ] + } + }, + { + "case_id": "hugo-479fe6c65493", + "repository": "hugo", + "commit": "479fe6c654937a850b65e74551dc4e857d52898f", + "base_revision": "81a5cdca0788ca39574a17d444c9db29d0b19e27", + "authored_at": "2026-04-01T13:54:10+02:00", + "query": "Fix potential content XSS by escaping dangerous URLs in links and images", + "query_tokens": [ + "content", + "dangerous", + "escaping", + "images", + "links", + "potential", + "urls" + ], + "specific_tokens": [ + "content", + "dangerous", + "escaping", + "images", + "links", + "potential", + "urls" + ], + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-7a43b928a67c", + "repository": "hugo", + "commit": "7a43b928a67c4dbb750a905230b3a66a4f242e98", + "base_revision": "555dfa207a6a631e4c8193cb6be82f54c750a8dc", + "authored_at": "2025-11-25T15:37:24+01:00", + "query": "Fix slow server startup of very big content trees", + "query_tokens": [ + "content", + "server", + "slow", + "startup", + "trees", + "very" + ], + "specific_tokens": [ + "content", + "server", + "slow", + "startup", + "trees", + "very" + ], + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.25, + "shared_tokens": [ + { + "path": "commands/server.go", + "tokens": [ + "server" + ] + } + ] + } + }, + { + "case_id": "hugo-8a979d54d1e5", + "repository": "hugo", + "commit": "8a979d54d1e5b3dbd68272413249fd7ff36b6c81", + "base_revision": "fd49df8f7a1152a6d303701a92d882a6d82dfce1", + "authored_at": "2026-01-28T23:50:57+08:00", + "query": "Fix data race when clearing cache in cachebusters", + "query_tokens": [ + "cache", + "cachebusters", + "clearing", + "data", + "race" + ], + "specific_tokens": [ + "cache", + "cachebusters", + "clearing", + "data", + "race" + ], + "required_files": [ + "config/commonConfig.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-8b00030b344d", + "repository": "hugo", + "commit": "8b00030b344d21635e5634698700f1d13145fead", + "base_revision": "c48551677c2504e3f2d7fa53ee42766e0333b957", + "authored_at": "2026-04-08T13:30:14+02:00", + "query": "Fix panic when passthrough elements are used in headings", + "query_tokens": [ + "elements", + "headings", + "panic", + "passthrough", + "used" + ], + "specific_tokens": [ + "elements", + "headings", + "panic", + "passthrough", + "used" + ], + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-8d6145f3c315", + "repository": "hugo", + "commit": "8d6145f3c31528ecafca4e5f301abd1c672468fd", + "base_revision": "ce2a156a4e89849244862705395791590d12b50a", + "authored_at": "2026-04-15T19:11:10+02:00", + "query": "Fix filename dimension identifiers (_role_X_, _version_X_) to replace mount config", + "query_tokens": [ + "config", + "dimension", + "filename", + "identifiers", + "mount", + "replace", + "role", + "version" + ], + "specific_tokens": [ + "config", + "dimension", + "filename", + "identifiers", + "mount", + "replace", + "role", + "version" + ], + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-8dfcece80541", + "repository": "hugo", + "commit": "8dfcece8054179ec869ceb775d10d64bbf08d6d5", + "base_revision": "9a6bfe2667f4e59e3cb78d3e70f75e6cb0fd1f95", + "authored_at": "2026-01-26T21:38:50+01:00", + "query": "Fix recently introduced partial rendering bug", + "query_tokens": [ + "introduced", + "partial", + "recently", + "rendering" + ], + "specific_tokens": [ + "introduced", + "partial", + "recently", + "rendering" + ], + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-95e5e9f4ab6e", + "repository": "hugo", + "commit": "95e5e9f4ab6e2b4565be4e46507aadeada2435a6", + "base_revision": "a00b5c72ac57afe26df6688ece3ca544a56df372", + "authored_at": "2026-06-11T11:57:09+02:00", + "query": "Fix multi --renderSegments merge behavior", + "query_tokens": [ + "behavior", + "merge", + "multi", + "render", + "segments" + ], + "specific_tokens": [ + "behavior", + "merge", + "multi", + "render", + "segments" + ], + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.667, + "shared_tokens": [ + { + "path": "hugolib/segments/segments.go", + "tokens": [ + "segments" + ] + }, + { + "path": "hugolib/segments/segments_integration_test.go", + "tokens": [ + "segments" + ] + } + ] + } + }, + { + "case_id": "hugo-974772422258", + "repository": "hugo", + "commit": "974772422258f1e8aab1c7603fc09bbb1a2c2571", + "base_revision": "a17bdbc5face02ed2470895bcbfe67db7749dc73", + "authored_at": "2026-04-14T16:33:38+02:00", + "query": "Fix it so we never auto-fallback to page resources in other roles/versions", + "query_tokens": [ + "auto", + "fallback", + "page", + "resources", + "roles", + "versions" + ], + "specific_tokens": [ + "auto", + "fallback", + "page", + "resources", + "roles", + "versions" + ], + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.429, + "shared_tokens": [ + { + "path": "hugolib/content_map_page.go", + "tokens": [ + "page" + ] + }, + { + "path": "hugolib/content_map_page_assembler.go", + "tokens": [ + "page" + ] + }, + { + "path": "resources/page/pagemeta/page_frontmatter.go", + "tokens": [ + "page", + "resources" + ] + } + ] + } + }, + { + "case_id": "hugo-a77548830407", + "repository": "hugo", + "commit": "a77548830407851d3a8d24085a7f5dc131ca886e", + "base_revision": "66ba63cd5f30ef8807fc524dbc67399376299bb1", + "authored_at": "2026-01-11T15:30:33+01:00", + "query": "Fix some default site redirect woes", + "query_tokens": [ + "default", + "redirect", + "site", + "woes" + ], + "specific_tokens": [ + "default", + "redirect", + "site", + "woes" + ], + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "hugolib/site.go", + "tokens": [ + "site" + ] + }, + { + "path": "hugolib/site_render.go", + "tokens": [ + "site" + ] + } + ] + } + }, + { + "case_id": "hugo-a808f6e40d4c", + "repository": "hugo", + "commit": "a808f6e40d4cb109c65d232ec63c5433bc180aeb", + "base_revision": "94f3908ec6a1618361ef07c61ac0402b7d21e07a", + "authored_at": "2026-08-08T13:36:46+02:00", + "query": "Fix server errors when deleting static files or directories", + "query_tokens": [ + "deleting", + "directories", + "errors", + "files", + "server", + "static" + ], + "specific_tokens": [ + "deleting", + "directories", + "files", + "server", + "static" + ], + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "testscripts/server/server__delete_static_file_mount.txt", + "testscripts/server/server__delete_static_mount_dir.txt" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-b5d43cdc1783", + "repository": "hugo", + "commit": "b5d43cdc1783796d9c6b17c7e135fa46d8b0279d", + "base_revision": "6ef8017f60117ad9d900cc59f10a962cd68566d6", + "authored_at": "2026-01-29T22:53:13+01:00", + "query": "Fix image DecodeConfig regression of WebP images from file cache", + "query_tokens": [ + "cache", + "config", + "decode", + "file", + "image", + "images", + "regression" + ], + "specific_tokens": [ + "cache", + "config", + "decode", + "file", + "image", + "images", + "regression" + ], + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "resources/image.go", + "tokens": [ + "image" + ] + }, + { + "path": "resources/image_cache.go", + "tokens": [ + "cache", + "image" + ] + }, + { + "path": "resources/images/images_integration_test.go", + "tokens": [ + "images" + ] + } + ] + } + }, + { + "case_id": "hugo-b82e496c130f", + "repository": "hugo", + "commit": "b82e496c130fa18eab4f3333b9b88e3823e63e45", + "base_revision": "33542d39663e3ec4140b5b425bdc4e40dec1b318", + "authored_at": "2025-12-17T17:21:11+01:00", + "query": "Fix some outdated front matter", + "query_tokens": [ + "front", + "matter", + "outdated" + ], + "specific_tokens": [ + "front", + "matter", + "outdated" + ], + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-c1b2e58bfb94", + "repository": "hugo", + "commit": "c1b2e58bfb9420aa92562cdbfa40de782b788234", + "base_revision": "77214117ae76ddac6b82dfa232b6aa5d85e64f6b", + "authored_at": "2026-01-20T11:00:27+01:00", + "query": "Fix file mount specifity issue within the same module", + "query_tokens": [ + "file", + "issue", + "module", + "mount", + "specifity", + "within" + ], + "specific_tokens": [ + "file", + "module", + "mount", + "specifity", + "within" + ], + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-c48551677c25", + "repository": "hugo", + "commit": "c48551677c2504e3f2d7fa53ee42766e0333b957", + "base_revision": "161d0d475773c5251ec00cc8848f1a174393155c", + "authored_at": "2026-04-08T12:07:37+02:00", + "query": "Fix panic on edit of legacy mapped template names that's also a valid path in the new setup", + "query_tokens": [ + "also", + "edit", + "legacy", + "mapped", + "names", + "panic", + "path", + "setup", + "template", + "valid" + ], + "specific_tokens": [ + "edit", + "legacy", + "mapped", + "names", + "panic", + "path", + "setup", + "template", + "valid" + ], + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-c7b35c8704ac", + "repository": "hugo", + "commit": "c7b35c8704ac8283d77afc5b13a52529363ed858", + "base_revision": "5916b61be94e9c552173038853a37c346eabae6b", + "authored_at": "2026-01-27T20:11:22+01:00", + "query": "Fix panic reported in discourse", + "query_tokens": [ + "discourse", + "panic", + "reported" + ], + "specific_tokens": [ + "discourse", + "panic", + "reported" + ], + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-ce009e3aa9fd", + "repository": "hugo", + "commit": "ce009e3aa9fdc0e9f8d2d209d0d4df591707a094", + "base_revision": "0755872424182a99520ab27f54df29ba2bcccddc", + "authored_at": "2026-04-06T18:01:22+02:00", + "query": "Fix auto-creation of root sections in multilingual sites", + "query_tokens": [ + "auto", + "creation", + "multilingual", + "root", + "sections", + "sites" + ], + "specific_tokens": [ + "auto", + "creation", + "multilingual", + "root", + "sections", + "sites" + ], + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-d4c0e4452c22", + "repository": "hugo", + "commit": "d4c0e4452c22fcb5eab457492d989793722379ac", + "base_revision": "418156ec3e943f4c8e00b33b896b759a2b942b03", + "authored_at": "2025-12-25T12:03:55+01:00", + "query": "Fix error with _content.gotmpl file with index.md siblings", + "query_tokens": [ + "content", + "error", + "file", + "gotmpl", + "index", + "siblings" + ], + "specific_tokens": [ + "content", + "file", + "gotmpl", + "index", + "siblings" + ], + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-e2e64aeec57c", + "repository": "hugo", + "commit": "e2e64aeec57ca333618a07d2277b6519d66f3871", + "base_revision": "9e7182e9e6a21235a025c6672072b1d4e1359368", + "authored_at": "2025-12-05T11:33:55+01:00", + "query": "Fix server rebuilds on editing content with Chinese terms", + "query_tokens": [ + "chinese", + "content", + "editing", + "rebuilds", + "server", + "terms" + ], + "specific_tokens": [ + "chinese", + "content", + "editing", + "rebuilds", + "server", + "terms" + ], + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "testscripts/commands/server__edit_content_with_chinese_tag.txt" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.25, + "shared_tokens": [ + { + "path": "commands/server.go", + "tokens": [ + "server" + ] + } + ] + } + }, + { + "case_id": "hugo-e9b9b36f46cc", + "repository": "hugo", + "commit": "e9b9b36f46ccc6250ffa72188c8a5c52f96b119e", + "base_revision": "b8a2c10d06d815cdaf61ef2f03fb37267612da2e", + "authored_at": "2026-01-02T16:22:25+01:00", + "query": "Fix alpha/fuzzy border issue with new webp decoder for images with with transparent background", + "query_tokens": [ + "alpha", + "background", + "border", + "decoder", + "fuzzy", + "images", + "issue", + "transparent", + "webp" + ], + "specific_tokens": [ + "alpha", + "background", + "border", + "decoder", + "fuzzy", + "images", + "transparent", + "webp" + ], + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.png", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.png" + ], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 1, + "shared_tokens": [ + { + "path": "internal/warpc/webp.go", + "tokens": [ + "webp" + ] + }, + { + "path": "resources/images/images_golden_integration_test.go", + "tokens": [ + "images" + ] + }, + { + "path": "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "tokens": [ + "images", + "webp" + ] + }, + { + "path": "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp", + "tokens": [ + "images", + "webp" + ] + } + ] + } + }, + { + "case_id": "hugo-f740d7cfe0eb", + "repository": "hugo", + "commit": "f740d7cfe0ebaec7056a8fc2b770e46b86c13ee7", + "base_revision": "5f46da6e2a3f838b40176775c0b46855f4f8f529", + "authored_at": "2025-12-20T11:26:33+01:00", + "query": "Fix panic when 404 is backed by a content file", + "query_tokens": [ + "backed", + "content", + "file", + "panic" + ], + "specific_tokens": [ + "backed", + "content", + "file", + "panic" + ], + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": true, + "path_leak_ratio": 0.333, + "shared_tokens": [ + { + "path": "hugolib/content_map_page_assembler.go", + "tokens": [ + "content" + ] + } + ] + } + }, + { + "case_id": "hugo-f772998fa349", + "repository": "hugo", + "commit": "f772998fa34927f469c70b83acdc72b62895d13e", + "base_revision": "8a55df7af2e6da31297245cc54fa2e3b521d93e8", + "authored_at": "2026-08-10T18:10:06+02:00", + "query": "Fix resource transformation chaining after content access", + "query_tokens": [ + "access", + "chaining", + "content", + "resource", + "transformation" + ], + "specific_tokens": [ + "access", + "chaining", + "content", + "resource", + "transformation" + ], + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-f797f8490229", + "repository": "hugo", + "commit": "f797f84902297b7dd37c3f39ce057789a4d8e950", + "base_revision": "e8577771c3275d16b7d1b31cca5e9fd3ea517416", + "authored_at": "2026-02-27T10:21:12+01:00", + "query": "Fix index out of range panic in fileEventsContentPaths", + "query_tokens": [ + "content", + "events", + "file", + "index", + "panic", + "paths", + "range" + ], + "specific_tokens": [ + "content", + "events", + "file", + "index", + "panic", + "paths", + "range" + ], + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + }, + { + "case_id": "hugo-f8b5fa09a649", + "repository": "hugo", + "commit": "f8b5fa09a64950c32b803821ede411ebfe772b7a", + "base_revision": "86fbb0f7a8bbb93e2e916390de9e5a4f24bf9f50", + "authored_at": "2026-05-11T12:52:53+02:00", + "query": "Fix prevention of direct symlink reads in resources.Get", + "query_tokens": [ + "direct", + "prevention", + "reads", + "resources", + "symlink" + ], + "specific_tokens": [ + "direct", + "prevention", + "reads", + "resources", + "symlink" + ], + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "derivable_files": [], + "incidental_files": [], + "excluded_files": [ + "testscripts/commands/no_symlinks.txt" + ], + "retrieval": { + "path_leak": false, + "path_leak_ratio": 0, + "shared_tokens": [] + } + } + ], + "rejected_sample": [ + { + "commit": "7b5199fdeff7d7682e9f979f468d55bc7b67b934", + "subject": "all: Run modernize -fix ./...", + "reason": "not-a-fix" + }, + { + "commit": "e31ff547d24c35edfa9d22bdd48b6cdf9688cf11", + "subject": "Upgrade to Go 1.27", + "reason": "not-a-fix" + }, + { + "commit": "87260e4a602c6b65134e0b6e1c4ccfd8ade7ee56", + "subject": "commands: Fix lang flag description in config command", + "reason": "not-a-fix" + }, + { + "commit": "8405b802cf647b88e3de872d6fb01323107aaf9b", + "subject": "tpl: Improve the return keyword in templates", + "reason": "not-a-fix" + }, + { + "commit": "bf05832d14f9916c8c8268b20fc0fa40f61c9f9f", + "subject": "page: Add IndexOf method to Pages", + "reason": "not-a-fix" + }, + { + "commit": "a05736cb9f6bbc930eb5815e1496ecccb1f94bf6", + "subject": "tpl/resources: Add resources.Publish", + "reason": "not-a-fix" + }, + { + "commit": "49dceb19f54610474bc6cf0417d41504c0b80a44", + "subject": "hugolib: Fix slice bounds panic when deleting multiple nodes at same path", + "reason": "not-a-fix" + }, + { + "commit": "5e7099256e148074dbc2be29d380a2b7c659f39c", + "subject": "hugolib: Fix ReadingTime and FuzzyWordCount calculations", + "reason": "not-a-fix" + }, + { + "commit": "0805c734a41b75403e3970e0070227916b6845d2", + "subject": "releaser: Prepare repository for 0.166.0-DEV", + "reason": "not-a-fix" + }, + { + "commit": "76a5e1880ab46688155b02e99bab9be2a6134492", + "subject": "releaser: Bump versions for release of 0.165.0", + "reason": "not-a-fix" + }, + { + "commit": "0bb337b22ab7953615ec151cb4cb454db7e08968", + "subject": "build(deps): bump github.com/bep/imagemeta from 0.17.3 to 1.0.0", + "reason": "not-a-fix" + }, + { + "commit": "03dc9170b3e410f50f2e74fdd4428865a5310e61", + "subject": "build(deps): bump github.com/evanw/esbuild from 0.28.1 to 0.28.2", + "reason": "not-a-fix" + }, + { + "commit": "c829b736cc5736a44f444a2c93f820b3068121c5", + "subject": "build(deps): bump github.com/tdewolff/minify/v2 from 2.24.14 to 2.24.16", + "reason": "not-a-fix" + }, + { + "commit": "995a2159e5a1b87a1321e154543e6ca20f9bb815", + "subject": "resources: Resume chained resource transformations", + "reason": "not-a-fix" + }, + { + "commit": "f88f0a9f59f302fb7c60d58921d636ed8da27aca", + "subject": "resources/jsconfig: Drop source root mapping for the current source root", + "reason": "not-a-fix" + }, + { + "commit": "dd3f2731e0612b3b402d1464d5612fd9fc342748", + "subject": "Remove Star History from README", + "reason": "not-a-fix" + }, + { + "commit": "8a55df7af2e6da31297245cc54fa2e3b521d93e8", + "subject": "Remove tailwindcss from the default security.exec.allow list (note)", + "reason": "not-a-fix" + }, + { + "commit": "52c9bd7908b4d02d4d0ff8f82a888834d6ee10d2", + "subject": "circleci: Upgrade to Go 1.26.5", + "reason": "not-a-fix" + }, + { + "commit": "44da086082a1dff487f4e5639bb03a65d9dd1ade", + "subject": "Add Data.Artifacts to css.Build and js.Build", + "reason": "not-a-fix" + }, + { + "commit": "94f3908ec6a1618361ef07c61ac0402b7d21e07a", + "subject": "build(deps): bump github.com/getkin/kin-openapi from 0.145.0 to 0.146.0", + "reason": "not-a-fix" + } + ] +} diff --git a/benchmarks/context-retrieval/results/A-coco.json b/benchmarks/context-retrieval/results/A-coco.json new file mode 100644 index 00000000..e697dcde --- /dev/null +++ b/benchmarks/context-retrieval/results/A-coco.json @@ -0,0 +1,2193 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "corpus_counts": { + "cases": 76, + "path_leak": 23, + "baseline_blind": 53, + "multi_file": 54 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "providers": [ + { + "provider_id": "cocoindex-code", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.193, + "full_recall_at_5": 0.0789, + "mean_recall_at_10": 0.2686, + "full_recall_at_10": 0.1447, + "mean_recall_at_20": 0.2752, + "full_recall_at_20": 0.1579, + "mean_recall_at_1000_tokens": 0.057, + "mean_recall_at_4000_tokens": 0.2193, + "mean_recall_at_16000_tokens": 0.2752, + "mean_precision_at_10": 0.0666, + "zero_hit_rate": 0.6053, + "median_tokens_delivered": 5196.5, + "median_latency_ms": 937.256, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 65, + "mean_recall_at_5": 0.1026, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1449, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1526, + "full_recall_at_20": 0.0154, + "mean_recall_at_1000_tokens": 0.0205, + "mean_recall_at_4000_tokens": 0.1179, + "mean_recall_at_16000_tokens": 0.1526, + "mean_precision_at_10": 0.0449, + "zero_hit_rate": 0.7077, + "median_tokens_delivered": 5202, + "median_latency_ms": 935.219, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 11, + "mean_recall_at_5": 0.7273, + "full_recall_at_5": 0.5455, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.2727, + "mean_recall_at_4000_tokens": 0.8182, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1947, + "zero_hit_rate": 0, + "median_tokens_delivered": 5191, + "median_latency_ms": 941.366, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.2681, + "full_recall_at_5": 0.1739, + "mean_recall_at_10": 0.3007, + "full_recall_at_10": 0.1739, + "mean_recall_at_20": 0.3007, + "full_recall_at_20": 0.1739, + "mean_recall_at_1000_tokens": 0.1014, + "mean_recall_at_4000_tokens": 0.2681, + "mean_recall_at_16000_tokens": 0.3007, + "mean_precision_at_10": 0.0686, + "zero_hit_rate": 0.5652, + "median_tokens_delivered": 5202, + "median_latency_ms": 927.29, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.1604, + "full_recall_at_5": 0.0377, + "mean_recall_at_10": 0.2547, + "full_recall_at_10": 0.1321, + "mean_recall_at_20": 0.2642, + "full_recall_at_20": 0.1509, + "mean_recall_at_1000_tokens": 0.0377, + "mean_recall_at_4000_tokens": 0.1981, + "mean_recall_at_16000_tokens": 0.2642, + "mean_precision_at_10": 0.0657, + "zero_hit_rate": 0.6226, + "median_tokens_delivered": 5191, + "median_latency_ms": 939.48, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 44362.5, + "peak_rss_mb": 51456, + "delta_rss_mb": 7093.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "cocoindex-code", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 332, + "latency_ms": 925.96, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 332, + "latency_ms": 887.048, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 332, + "latency_ms": 894.944, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4653, + "latency_ms": 1039.135, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5487, + "latency_ms": 962.306, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5108, + "latency_ms": 1041.316, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5233, + "latency_ms": 1043.446, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5482, + "latency_ms": 916.746, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5202, + "latency_ms": 962.546, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 4698, + "latency_ms": 917.515, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 5434, + "latency_ms": 1193.146, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 5146, + "latency_ms": 966.077, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5317, + "latency_ms": 940.893, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5553, + "latency_ms": 1020.298, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4820, + "latency_ms": 927.29, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5667, + "latency_ms": 970.47, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5230, + "latency_ms": 938.818, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 5402, + "latency_ms": 940.006, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5352, + "latency_ms": 939.48, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4931, + "latency_ms": 919.663, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 4745, + "latency_ms": 932.236, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5626, + "latency_ms": 943.262, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5131, + "latency_ms": 1174.097, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5774, + "latency_ms": 972.963, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4810, + "latency_ms": 937.081, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4776, + "latency_ms": 886.681, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 5347, + "latency_ms": 968.098, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4514, + "latency_ms": 1007.365, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "test_helpers.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 5700, + "latency_ms": 941.514, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5045, + "latency_ms": 884.329, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 5814, + "latency_ms": 915.462, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 5502, + "latency_ms": 929.807, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5392, + "latency_ms": 945.235, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5599, + "latency_ms": 921.311, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5488, + "latency_ms": 968.13, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4909, + "latency_ms": 921.311, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5371, + "latency_ms": 884.461, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5202, + "latency_ms": 912.07, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.125, + "recall_at_20": 0.25, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 7, + "tokens_delivered": 5314, + "latency_ms": 925.115, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5103, + "latency_ms": 914.091, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3486, + "latency_ms": 944.228, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 5654, + "latency_ms": 912.342, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2913, + "latency_ms": 906.122, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5140, + "latency_ms": 955.815, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4887, + "latency_ms": 941.366, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5440, + "latency_ms": 915.157, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5346, + "latency_ms": 942.272, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5191, + "latency_ms": 1000.142, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5433, + "latency_ms": 1026.906, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 5371, + "latency_ms": 911.037, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4656, + "latency_ms": 930.027, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5353, + "latency_ms": 920.389, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4870, + "latency_ms": 940.248, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5421, + "latency_ms": 913.67, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5344, + "latency_ms": 961.673, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 5075, + "latency_ms": 932.382, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5077, + "latency_ms": 930.983, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 5081, + "latency_ms": 910.888, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4952, + "latency_ms": 905.832, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 5041, + "latency_ms": 909.716, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 5727, + "latency_ms": 904.404, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4991, + "latency_ms": 959.509, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 5121, + "latency_ms": 990.799, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 4729, + "latency_ms": 1492.182, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4821, + "latency_ms": 911.735, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "githubapi_test.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 5462, + "latency_ms": 927.511, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 5661, + "latency_ms": 937.431, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5280, + "latency_ms": 998.016, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5461, + "latency_ms": 932.302, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4736, + "latency_ms": 925.72, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5709, + "latency_ms": 928.223, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 5179, + "latency_ms": 941.948, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5252, + "latency_ms": 938.479, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4937, + "latency_ms": 945.851, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug_test.go" + ], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 3667, + "latency_ms": 935.219, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5050, + "latency_ms": 962.648, + "indexed_revision_matches": true + } + ], + "tier": "small", + "tier_code_files": 99, + "tier_provenance": "Stamped after the fact from a measured file count; the run predates tier stamping. Tier is a property of the repository, not of the run." +} diff --git a/benchmarks/context-retrieval/results/A-gitnexus.json b/benchmarks/context-retrieval/results/A-gitnexus.json new file mode 100644 index 00000000..e818837c --- /dev/null +++ b/benchmarks/context-retrieval/results/A-gitnexus.json @@ -0,0 +1,2196 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "corpus_counts": { + "cases": 76, + "path_leak": 23, + "baseline_blind": 53, + "multi_file": 54 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "providers": [ + { + "provider_id": "gitnexus", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.4309, + "full_recall_at_5": 0.2895, + "mean_recall_at_10": 0.5428, + "full_recall_at_10": 0.4079, + "mean_recall_at_20": 0.6743, + "full_recall_at_20": 0.5395, + "mean_recall_at_1000_tokens": 0.0241, + "mean_recall_at_4000_tokens": 0.1283, + "mean_recall_at_16000_tokens": 0.6075, + "mean_precision_at_10": 0.1635, + "zero_hit_rate": 0.1842, + "median_tokens_delivered": 16385, + "median_latency_ms": 14818.2285, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 45, + "mean_recall_at_5": 0.1611, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2278, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.45, + "full_recall_at_20": 0.2222, + "mean_recall_at_1000_tokens": 0.0185, + "mean_recall_at_4000_tokens": 0.0722, + "mean_recall_at_16000_tokens": 0.3926, + "mean_precision_at_10": 0.0832, + "zero_hit_rate": 0.3111, + "median_tokens_delivered": 16385, + "median_latency_ms": 15056.469, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 31, + "mean_recall_at_5": 0.8226, + "full_recall_at_5": 0.7097, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.0323, + "mean_recall_at_4000_tokens": 0.2097, + "mean_recall_at_16000_tokens": 0.9194, + "mean_precision_at_10": 0.28, + "zero_hit_rate": 0, + "median_tokens_delivered": 16385, + "median_latency_ms": 14711.967, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.413, + "full_recall_at_5": 0.1739, + "mean_recall_at_10": 0.558, + "full_recall_at_10": 0.3478, + "mean_recall_at_20": 0.8188, + "full_recall_at_20": 0.6522, + "mean_recall_at_1000_tokens": 0.058, + "mean_recall_at_4000_tokens": 0.1884, + "mean_recall_at_16000_tokens": 0.7283, + "mean_precision_at_10": 0.1564, + "zero_hit_rate": 0.0435, + "median_tokens_delivered": 16385, + "median_latency_ms": 15248.372, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.4387, + "full_recall_at_5": 0.3396, + "mean_recall_at_10": 0.5362, + "full_recall_at_10": 0.434, + "mean_recall_at_20": 0.6116, + "full_recall_at_20": 0.4906, + "mean_recall_at_1000_tokens": 0.0094, + "mean_recall_at_4000_tokens": 0.1022, + "mean_recall_at_16000_tokens": 0.555, + "mean_precision_at_10": 0.1665, + "zero_hit_rate": 0.2453, + "median_tokens_delivered": 16385, + "median_latency_ms": 14661.018, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 30581.6, + "peak_rss_mb": 41675.3, + "delta_rss_mb": 11093.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "gitnexus", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go", + "logger_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1429, + "recall_at_20": 0.3333, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 13454.628, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 18481.331, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 16069.202, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 12622, + "latency_ms": 18798.887, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 18168.075, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 21655.6, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 18558.695, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 10306, + "latency_ms": 17254.169, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1201, + "latency_ms": 15945.126, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 16011.905, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 14869.894, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 7018, + "latency_ms": 15812.673, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 12419, + "latency_ms": 18118.897, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 13251.539, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 16385, + "latency_ms": 17340.146, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 16762.733, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 14418.072, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 13245.587, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 12784.281, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 12785.691, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5497, + "latency_ms": 13444.356, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 13980.665, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 16833.358, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 16385, + "latency_ms": 16691.765, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 16216.027, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 12514.615, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 14291.459, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "binding/default_validator.go", + "gin.go" + ], + "missed": [ + "context_test.go", + "errors.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2222, + "recall_at_20": 0.3333, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 13748.927, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "test_helpers.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 12738.194, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 11089, + "latency_ms": 13441.784, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 14815.056, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 14661.018, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 15522.5, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 16382, + "latency_ms": 15792.471, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 16385, + "latency_ms": 12816.914, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 49153, + "latency_ms": 13671.018, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 16289.335, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 12600, + "latency_ms": 13471.434, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 17064, + "latency_ms": 13869.359, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 13665.026, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go" + ], + "missed": [ + "mode_test.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 8185, + "latency_ms": 15116.989, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 13328.285, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 20472, + "latency_ms": 13444.141, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 12964.272, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go" + ], + "missed": [ + "binding/validate_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 7744, + "latency_ms": 13410.314, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 16385, + "latency_ms": 14603.66, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 13219.394, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 14821.401, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 13594, + "latency_ms": 14732.744, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5363, + "latency_ms": 15307.045, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 14655.944, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 10483, + "latency_ms": 14612.491, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 12204.37, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 15740, + "latency_ms": 16117.934, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 16385, + "latency_ms": 16131.247, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 16076.914, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 23164.481, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 16385, + "latency_ms": 18137.104, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go" + ], + "missed": [ + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 9266, + "latency_ms": 18674.385, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 16385, + "latency_ms": 18194.492, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 16385, + "latency_ms": 16472.981, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 14106.744, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 16231.752, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 14711.967, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10787, + "latency_ms": 15056.469, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 16385, + "latency_ms": 12080.775, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 12399.339, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 8987, + "latency_ms": 15119.777, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 15785.833, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 16385, + "latency_ms": 13719.339, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 7750, + "latency_ms": 14161.465, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 13338.552, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 9142, + "latency_ms": 16100.716, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 21703.638, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 9255, + "latency_ms": 14268.853, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 15248.372, + "indexed_revision_matches": true + } + ], + "tier": "small", + "tier_code_files": 99, + "tier_provenance": "Stamped after the fact from a measured file count; the run predates tier stamping. Tier is a property of the repository, not of the run." +} diff --git a/benchmarks/context-retrieval/results/A-gortex.json b/benchmarks/context-retrieval/results/A-gortex.json new file mode 100644 index 00000000..4d673dee --- /dev/null +++ b/benchmarks/context-retrieval/results/A-gortex.json @@ -0,0 +1,2188 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "corpus_counts": { + "cases": 76, + "path_leak": 23, + "baseline_blind": 53, + "multi_file": 54 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "providers": [ + { + "provider_id": "gortex", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.4254, + "full_recall_at_5": 0.3421, + "mean_recall_at_10": 0.5219, + "full_recall_at_10": 0.4342, + "mean_recall_at_20": 0.5592, + "full_recall_at_20": 0.4737, + "mean_recall_at_1000_tokens": 0.3947, + "mean_recall_at_4000_tokens": 0.5548, + "mean_recall_at_16000_tokens": 0.5592, + "mean_precision_at_10": 0.1992, + "zero_hit_rate": 0.3553, + "median_tokens_delivered": 888, + "median_latency_ms": 17579.429, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 43, + "mean_recall_at_5": 0.1046, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.155, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2209, + "full_recall_at_20": 0.0698, + "mean_recall_at_1000_tokens": 0.1163, + "mean_recall_at_4000_tokens": 0.2209, + "mean_recall_at_16000_tokens": 0.2209, + "mean_precision_at_10": 0.0893, + "zero_hit_rate": 0.6279, + "median_tokens_delivered": 547, + "median_latency_ms": 17619.528, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 33, + "mean_recall_at_5": 0.8434, + "full_recall_at_5": 0.7879, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.7576, + "mean_recall_at_4000_tokens": 0.9899, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.3424, + "zero_hit_rate": 0, + "median_tokens_delivered": 1189, + "median_latency_ms": 17539.33, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.558, + "full_recall_at_5": 0.4783, + "mean_recall_at_10": 0.6884, + "full_recall_at_10": 0.6087, + "mean_recall_at_20": 0.7246, + "full_recall_at_20": 0.6522, + "mean_recall_at_1000_tokens": 0.5, + "mean_recall_at_4000_tokens": 0.7101, + "mean_recall_at_16000_tokens": 0.7246, + "mean_precision_at_10": 0.1804, + "zero_hit_rate": 0.2174, + "median_tokens_delivered": 1296, + "median_latency_ms": 19890.19, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3679, + "full_recall_at_5": 0.283, + "mean_recall_at_10": 0.4497, + "full_recall_at_10": 0.3585, + "mean_recall_at_20": 0.4874, + "full_recall_at_20": 0.3962, + "mean_recall_at_1000_tokens": 0.3491, + "mean_recall_at_4000_tokens": 0.4874, + "mean_recall_at_16000_tokens": 0.4874, + "mean_precision_at_10": 0.2074, + "zero_hit_rate": 0.4151, + "median_tokens_delivered": 812, + "median_latency_ms": 17240.379, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 39382.8, + "peak_rss_mb": 45202.7, + "delta_rss_mb": 5819.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 1982464 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "gortex", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [ + "logger_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 2433, + "latency_ms": 12506.079, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 2497, + "latency_ms": 10673.679, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2052, + "latency_ms": 16236.896, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1332, + "latency_ms": 12743.27, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1116, + "latency_ms": 8029.55, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 721, + "latency_ms": 7161.456, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 812, + "latency_ms": 20264.545, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2169, + "latency_ms": 22866.643, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1056, + "latency_ms": 10010.987, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1296, + "latency_ms": 22086.838, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2544, + "latency_ms": 14051.956, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1407, + "latency_ms": 14448.219, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 437, + "latency_ms": 26784.971, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 922, + "latency_ms": 17435.489, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1346, + "latency_ms": 26230.583, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 498, + "latency_ms": 19743.003, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 347, + "latency_ms": 16986.875, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 715, + "latency_ms": 24403.17, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go" + ], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1667, + "recall_at_20": 0.3333, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 793, + "latency_ms": 8734.805, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 527, + "latency_ms": 24741.54, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1743, + "latency_ms": 9610.535, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 498, + "latency_ms": 20436.549, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44, + "latency_ms": 33097.578, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1424, + "latency_ms": 43820.296, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 248, + "latency_ms": 14910.621, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 547, + "latency_ms": 14801.568, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1183, + "latency_ms": 9388.815, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "gin.go", + "response_writer.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2222, + "recall_at_20": 0.3333, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 847, + "latency_ms": 26295.942, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.3, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 4572, + "latency_ms": 18238.455, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 436, + "latency_ms": 21809.335, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go" + ], + "missed": [ + "render/render_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 458, + "latency_ms": 23718.66, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1160, + "latency_ms": 17619.528, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 939, + "latency_ms": 30128.052, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 1613, + "latency_ms": 10274.788, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2196, + "latency_ms": 17692.625, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 473, + "latency_ms": 44471.819, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 776, + "latency_ms": 22353.157, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 736, + "latency_ms": 32039.198, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33, + "latency_ms": 25548.246, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1515, + "latency_ms": 17240.379, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go" + ], + "missed": [ + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 10, + "tokens_delivered": 2028, + "latency_ms": 6713.749, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1170, + "latency_ms": 15544.432, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 2220, + "latency_ms": 26727.858, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1982, + "latency_ms": 8941.781, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 251, + "latency_ms": 22667.015, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 3782, + "latency_ms": 6754.861, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1033, + "latency_ms": 17539.33, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 924, + "latency_ms": 19360.205, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 339, + "latency_ms": 14977.316, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 352, + "latency_ms": 16238.143, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 550, + "latency_ms": 19225.746, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 578, + "latency_ms": 32091.236, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 1043, + "latency_ms": 35223.025, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 470, + "latency_ms": 8257.506, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1189, + "latency_ms": 23812.389, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 462, + "latency_ms": 48973.851, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 491, + "latency_ms": 14210.623, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 1819, + "latency_ms": 16311.151, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 488, + "latency_ms": 9499.238, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1657, + "latency_ms": 25063.402, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1983, + "latency_ms": 10554.507, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 607, + "latency_ms": 14130.087, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1305, + "latency_ms": 77973.655, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 1903, + "latency_ms": 33079.98, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 160, + "latency_ms": 13481.87, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 672, + "latency_ms": 18471.42, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1123, + "latency_ms": 10311.642, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 766, + "latency_ms": 11614.613, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 899, + "latency_ms": 7083.517, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 472, + "latency_ms": 14112.778, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 916, + "latency_ms": 19890.19, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 877, + "latency_ms": 83409.246, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38, + "latency_ms": 45525.459, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 355, + "latency_ms": 6456.389, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 500, + "latency_ms": 27059.44, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 859, + "latency_ms": 10116.052, + "indexed_revision_matches": true + } + ], + "tier": "small", + "tier_code_files": 99, + "tier_provenance": "Stamped after the fact from a measured file count; the run predates tier stamping. Tier is a property of the repository, not of the run." +} diff --git a/benchmarks/context-retrieval/results/A-jcm.json b/benchmarks/context-retrieval/results/A-jcm.json new file mode 100644 index 00000000..3f1681d2 --- /dev/null +++ b/benchmarks/context-retrieval/results/A-jcm.json @@ -0,0 +1,2193 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "corpus_counts": { + "cases": 76, + "path_leak": 23, + "baseline_blind": 53, + "multi_file": 54 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "providers": [ + { + "provider_id": "jcodemunch", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.3377, + "full_recall_at_5": 0.2237, + "mean_recall_at_10": 0.3377, + "full_recall_at_10": 0.2237, + "mean_recall_at_20": 0.3377, + "full_recall_at_20": 0.2237, + "mean_recall_at_1000_tokens": 0.3246, + "mean_recall_at_4000_tokens": 0.3377, + "mean_recall_at_16000_tokens": 0.3377, + "mean_precision_at_10": 0.3114, + "zero_hit_rate": 0.5395, + "median_tokens_delivered": 481.5, + "median_latency_ms": 596.48, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 6 + }, + "baseline_missed": { + "cases": 59, + "mean_recall_at_5": 0.1469, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1469, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1469, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.1469, + "mean_recall_at_4000_tokens": 0.1469, + "mean_recall_at_16000_tokens": 0.1469, + "mean_precision_at_10": 0.1893, + "zero_hit_rate": 0.6949, + "median_tokens_delivered": 475, + "median_latency_ms": 596.003, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 6 + }, + "baseline_solved": { + "cases": 17, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.9412, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.7353, + "zero_hit_rate": 0, + "median_tokens_delivered": 502, + "median_latency_ms": 619.043, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.4058, + "full_recall_at_5": 0.3043, + "mean_recall_at_10": 0.4058, + "full_recall_at_10": 0.3043, + "mean_recall_at_20": 0.4058, + "full_recall_at_20": 0.3043, + "mean_recall_at_1000_tokens": 0.4058, + "mean_recall_at_4000_tokens": 0.4058, + "mean_recall_at_16000_tokens": 0.4058, + "mean_precision_at_10": 0.3406, + "zero_hit_rate": 0.4783, + "median_tokens_delivered": 482, + "median_latency_ms": 639.513, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 2 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3082, + "full_recall_at_5": 0.1887, + "mean_recall_at_10": 0.3082, + "full_recall_at_10": 0.1887, + "mean_recall_at_20": 0.3082, + "full_recall_at_20": 0.1887, + "mean_recall_at_1000_tokens": 0.2893, + "mean_recall_at_4000_tokens": 0.3082, + "mean_recall_at_16000_tokens": 0.3082, + "mean_precision_at_10": 0.2987, + "zero_hit_rate": 0.566, + "median_tokens_delivered": 480, + "median_latency_ms": 583.097, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 4 + } + }, + "resources": { + "baseline_rss_mb": 29127.8, + "peak_rss_mb": 45274.7, + "delta_rss_mb": 16147, + "scope": "system-wide-upper-bound", + "index_bytes_added": 58568704 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "jcodemunch", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1233, + "latency_ms": 481.127, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1333, + "latency_ms": 464.513, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 399, + "latency_ms": 479.76, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 481, + "latency_ms": 447.318, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1309, + "latency_ms": 446.551, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 471, + "latency_ms": 456.791, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 502, + "latency_ms": 521.475, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 631, + "latency_ms": 496.094, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 141, + "latency_ms": 452.105, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 537, + "latency_ms": 494.41, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 368, + "latency_ms": 501.594, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 502, + "latency_ms": 524.873, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 368, + "latency_ms": 553.2, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 402, + "latency_ms": 546.183, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1924, + "latency_ms": 528.753, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 520, + "latency_ms": 609.601, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 502, + "latency_ms": 511.164, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 414, + "latency_ms": 529.384, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 377, + "latency_ms": 558.29, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 514, + "latency_ms": 502.321, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 456, + "latency_ms": 493.464, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 400, + "latency_ms": 525.673, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1574, + "latency_ms": 532.796, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 515, + "latency_ms": 459.43, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 469, + "latency_ms": 453.724, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 435, + "latency_ms": 437.839, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 441, + "latency_ms": 499.062, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 649, + "latency_ms": 517.53, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [], + "missed": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 409, + "latency_ms": 496.416, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 532, + "latency_ms": 539.648, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 508, + "latency_ms": 631.157, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 480, + "latency_ms": 596.957, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1356, + "latency_ms": 974.035, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 475, + "latency_ms": 1157.779, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 838, + "latency_ms": 941.669, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 485, + "latency_ms": 1058.156, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 778, + "latency_ms": 1014.724, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 310, + "latency_ms": 1271.697, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 354, + "latency_ms": 1056.041, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 503, + "latency_ms": 1506.67, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "mode.go" + ], + "missed": [ + "debug.go", + "mode_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 482, + "latency_ms": 1100.245, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 361, + "latency_ms": 701.508, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1443, + "latency_ms": 666.895, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 444, + "latency_ms": 1042.42, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/validate_test.go" + ], + "missed": [ + "binding/default_validator.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1208, + "latency_ms": 596.003, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 467, + "latency_ms": 1010.677, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 488, + "latency_ms": 627.152, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1336, + "latency_ms": 661.354, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 375, + "latency_ms": 1734.985, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 432, + "latency_ms": 675.55, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 589, + "latency_ms": 936.433, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 790, + "latency_ms": 676.515, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 607, + "latency_ms": 784.095, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 486, + "latency_ms": 1248.404, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1505, + "latency_ms": 772.018, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 367, + "latency_ms": 619.043, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 483, + "latency_ms": 575.439, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 395, + "latency_ms": 549.192, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 516, + "latency_ms": 556.899, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 456, + "latency_ms": 1277.583, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 473, + "latency_ms": 1224.115, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 427, + "latency_ms": 1298.478, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 503, + "latency_ms": 639.513, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 439, + "latency_ms": 429.236, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 399, + "latency_ms": 614.203, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 385, + "latency_ms": 878.867, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 470, + "latency_ms": 750.395, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 438, + "latency_ms": 747.303, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 502, + "latency_ms": 795.678, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 258, + "latency_ms": 888.241, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 463, + "latency_ms": 696.148, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 579, + "latency_ms": 655.164, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/protobuf.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 374, + "latency_ms": 589.489, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1473, + "latency_ms": 583.097, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 418, + "latency_ms": 539.862, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 485, + "latency_ms": 549.444, + "indexed_revision_matches": true + } + ], + "tier": "small", + "tier_code_files": 99, + "tier_provenance": "Stamped after the fact from a measured file count; the run predates tier stamping. Tier is a property of the repository, not of the run." +} diff --git a/benchmarks/context-retrieval/results/A-tsregex.json b/benchmarks/context-retrieval/results/A-tsregex.json new file mode 100644 index 00000000..0e0a63ab --- /dev/null +++ b/benchmarks/context-retrieval/results/A-tsregex.json @@ -0,0 +1,2197 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "corpus_counts": { + "cases": 76, + "path_leak": 23, + "baseline_blind": 53, + "multi_file": 54 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "providers": [ + { + "provider_id": "token-savior-regex", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.3794, + "full_recall_at_5": 0.2632, + "mean_recall_at_10": 0.5164, + "full_recall_at_10": 0.3816, + "mean_recall_at_20": 0.523, + "full_recall_at_20": 0.3816, + "mean_recall_at_1000_tokens": 0.2939, + "mean_recall_at_4000_tokens": 0.523, + "mean_recall_at_16000_tokens": 0.523, + "mean_precision_at_10": 0.1804, + "zero_hit_rate": 0.3289, + "median_tokens_delivered": 1914.5, + "median_latency_ms": 1803.2805, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 47, + "mean_recall_at_5": 0.1454, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2181, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2287, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.117, + "mean_recall_at_4000_tokens": 0.2287, + "mean_recall_at_16000_tokens": 0.2287, + "mean_precision_at_10": 0.0927, + "zero_hit_rate": 0.5319, + "median_tokens_delivered": 1729, + "median_latency_ms": 1727.944, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 29, + "mean_recall_at_5": 0.7586, + "full_recall_at_5": 0.6897, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.5805, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.3227, + "zero_hit_rate": 0, + "median_tokens_delivered": 2121, + "median_latency_ms": 1875.013, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.4928, + "full_recall_at_5": 0.3478, + "mean_recall_at_10": 0.6087, + "full_recall_at_10": 0.4348, + "mean_recall_at_20": 0.6087, + "full_recall_at_20": 0.4348, + "mean_recall_at_1000_tokens": 0.3043, + "mean_recall_at_4000_tokens": 0.6087, + "mean_recall_at_16000_tokens": 0.6087, + "mean_precision_at_10": 0.2407, + "zero_hit_rate": 0.2174, + "median_tokens_delivered": 2296, + "median_latency_ms": 1815.403, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3302, + "full_recall_at_5": 0.2264, + "mean_recall_at_10": 0.4764, + "full_recall_at_10": 0.3585, + "mean_recall_at_20": 0.4858, + "full_recall_at_20": 0.3585, + "mean_recall_at_1000_tokens": 0.2893, + "mean_recall_at_4000_tokens": 0.4858, + "mean_recall_at_16000_tokens": 0.4858, + "mean_precision_at_10": 0.1543, + "zero_hit_rate": 0.3774, + "median_tokens_delivered": 1844, + "median_latency_ms": 1791.158, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 31868.3, + "peak_rss_mb": 38145, + "delta_rss_mb": 6276.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 512000 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "token-savior-regex", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2352, + "latency_ms": 1842.053, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1729, + "latency_ms": 1644.125, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2901, + "latency_ms": 1783.245, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1377, + "latency_ms": 1256.415, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 387, + "latency_ms": 1596.521, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1578, + "latency_ms": 1540.735, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 943, + "latency_ms": 1204.23, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1412, + "latency_ms": 2063.293, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 319, + "latency_ms": 1194.457, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2690, + "latency_ms": 1398.694, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2020, + "latency_ms": 1367.603, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1924, + "latency_ms": 2815.803, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1001, + "latency_ms": 1208.745, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2716, + "latency_ms": 2348.334, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 3172, + "latency_ms": 2850.55, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1844, + "latency_ms": 3065.53, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 297, + "latency_ms": 1711.08, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1328, + "latency_ms": 1147.367, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.375, + "recall_at_20": 1, + "precision_at_20": 0.375, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 982, + "latency_ms": 2605.353, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1196, + "latency_ms": 1727.944, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2815, + "latency_ms": 2184.782, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 2780, + "latency_ms": 2073.744, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 4038, + "latency_ms": 3493.931, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 1547, + "latency_ms": 1731.216, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1050, + "latency_ms": 1191.94, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1388, + "latency_ms": 3172.699, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 880, + "latency_ms": 1658.361, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1458, + "latency_ms": 1294.852, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "context_test.go", + "test_helpers.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1667, + "recall_at_20": 0.3333, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 3956, + "latency_ms": 2324.104, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1475, + "latency_ms": 1342.44, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [], + "missed": [ + "render/data.go", + "render/render_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3446, + "latency_ms": 3125.003, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2014, + "latency_ms": 3772.606, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3682, + "latency_ms": 3033.474, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 3014, + "latency_ms": 2493.939, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3416, + "latency_ms": 2844.377, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3054, + "latency_ms": 1970.138, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1603, + "latency_ms": 1875.013, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 850, + "latency_ms": 839.51, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 274, + "latency_ms": 942.109, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 2657, + "latency_ms": 2135.786, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go" + ], + "missed": [ + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 6, + "tokens_delivered": 2495, + "latency_ms": 2185.082, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2139, + "latency_ms": 2580.139, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 929, + "latency_ms": 1000.026, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2463, + "latency_ms": 1620.102, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1622, + "latency_ms": 1502.123, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2437, + "latency_ms": 2092.477, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3543, + "latency_ms": 3718.213, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3732, + "latency_ms": 2541.658, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1524, + "latency_ms": 1340.411, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1363, + "latency_ms": 1252.16, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2351, + "latency_ms": 3307.555, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 9, + "tokens_delivered": 3560, + "latency_ms": 4539.96, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1722, + "latency_ms": 1662.368, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1555, + "latency_ms": 1652.541, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2733, + "latency_ms": 2620.49, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1355, + "latency_ms": 1509.852, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1019, + "latency_ms": 1450.385, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1962, + "latency_ms": 1815.403, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3046, + "latency_ms": 2558.836, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2151, + "latency_ms": 1394.816, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1905, + "latency_ms": 984.307, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1087, + "latency_ms": 1020.064, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4937, + "latency_ms": 3959.587, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 2314, + "latency_ms": 2644.193, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1318, + "latency_ms": 1471.136, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 813, + "latency_ms": 2484.738, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1487, + "latency_ms": 1791.158, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 2778, + "latency_ms": 2650.703, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2121, + "latency_ms": 2893.211, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1878, + "latency_ms": 1773.247, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2296, + "latency_ms": 2479.904, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 2992, + "latency_ms": 3206.595, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3005, + "latency_ms": 1623.278, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1552, + "latency_ms": 2731.319, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2380, + "latency_ms": 1291.865, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1032, + "latency_ms": 979.334, + "indexed_revision_matches": true + } + ], + "tier": "small", + "tier_code_files": 99, + "tier_provenance": "Stamped after the fact from a measured file count; the run predates tier stamping. Tier is a property of the repository, not of the run." +} diff --git a/benchmarks/context-retrieval/results/A-tssem.json b/benchmarks/context-retrieval/results/A-tssem.json new file mode 100644 index 00000000..8ab63c20 --- /dev/null +++ b/benchmarks/context-retrieval/results/A-tssem.json @@ -0,0 +1,2182 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "corpus_counts": { + "cases": 76, + "path_leak": 23, + "baseline_blind": 53, + "multi_file": 54 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "providers": [ + { + "provider_id": "token-savior", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.1513, + "full_recall_at_5": 0.1053, + "mean_recall_at_10": 0.1513, + "full_recall_at_10": 0.1053, + "mean_recall_at_20": 0.1513, + "full_recall_at_20": 0.1053, + "mean_recall_at_1000_tokens": 0.1513, + "mean_recall_at_4000_tokens": 0.1513, + "mean_recall_at_16000_tokens": 0.1513, + "mean_precision_at_10": 0.1842, + "zero_hit_rate": 0.8026, + "median_tokens_delivered": 25, + "median_latency_ms": 20993.215, + "payload_kind": [ + "tool-output" + ], + "unavailable": 3 + }, + "baseline_missed": { + "cases": 68, + "mean_recall_at_5": 0.0515, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0515, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0515, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.0515, + "mean_recall_at_4000_tokens": 0.0515, + "mean_recall_at_16000_tokens": 0.0515, + "mean_precision_at_10": 0.0956, + "zero_hit_rate": 0.8971, + "median_tokens_delivered": 25, + "median_latency_ms": 20993.215, + "payload_kind": [ + "tool-output" + ], + "unavailable": 3 + }, + "baseline_solved": { + "cases": 8, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 1, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.9375, + "zero_hit_rate": 0, + "median_tokens_delivered": 87.5, + "median_latency_ms": 21610.8535, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.1957, + "full_recall_at_5": 0.1304, + "mean_recall_at_10": 0.1957, + "full_recall_at_10": 0.1304, + "mean_recall_at_20": 0.1957, + "full_recall_at_20": 0.1304, + "mean_recall_at_1000_tokens": 0.1957, + "mean_recall_at_4000_tokens": 0.1957, + "mean_recall_at_16000_tokens": 0.1957, + "mean_precision_at_10": 0.2609, + "zero_hit_rate": 0.7391, + "median_tokens_delivered": 25, + "median_latency_ms": 19738.733, + "payload_kind": [ + "tool-output" + ], + "unavailable": 1 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.1321, + "full_recall_at_5": 0.0943, + "mean_recall_at_10": 0.1321, + "full_recall_at_10": 0.0943, + "mean_recall_at_20": 0.1321, + "full_recall_at_20": 0.0943, + "mean_recall_at_1000_tokens": 0.1321, + "mean_recall_at_4000_tokens": 0.1321, + "mean_recall_at_16000_tokens": 0.1321, + "mean_precision_at_10": 0.1509, + "zero_hit_rate": 0.8302, + "median_tokens_delivered": 25, + "median_latency_ms": 21669.679, + "payload_kind": [ + "tool-output" + ], + "unavailable": 2 + } + }, + "resources": { + "baseline_rss_mb": 24972.3, + "peak_rss_mb": 33122.6, + "delta_rss_mb": 8150.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "token-savior", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67, + "latency_ms": 30061.243, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 30394.088, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 29500.279, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 22005.353, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69, + "latency_ms": 24696.162, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 81, + "latency_ms": 21826.016, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 89, + "latency_ms": 29670.585, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 29872.23, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 28703.918, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 73, + "latency_ms": 27321.234, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80, + "latency_ms": 21654.328, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 30349.165, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 76, + "latency_ms": 27872.615, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 21591.54, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 20332.099, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 20332.118, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 75, + "latency_ms": 20844.792, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 20647.155, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 21024.866, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 17309.486, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 83, + "latency_ms": 18814.454, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 18961.586, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 167, + "latency_ms": 26203.522, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 24254.493, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 94, + "latency_ms": 19985.332, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 18263.238, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 85, + "latency_ms": 18314.941, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 19331.976, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [], + "missed": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 19109.76, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 68, + "latency_ms": 23799.6, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [], + "missed": [ + "render/data.go", + "render/render_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 24248.107, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 79, + "latency_ms": 22380.587, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 18320.596, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 23767.254, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 17838.661, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 19738.733, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 72, + "latency_ms": 17610.348, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 134, + "latency_ms": 16926.865, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 20524.55, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 18621.032, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 17737.694, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 17145.669, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 18592.777, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 23679.32, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 18008.334, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 139, + "latency_ms": 19731.757, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 79, + "latency_ms": 20225.055, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 86, + "latency_ms": 17196.78, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 18100.15, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 87, + "latency_ms": 18354.163, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 23949.312, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 24354.499, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 88, + "latency_ms": 18553.194, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 17646.052, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 24302.776, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 18180.346, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 17959.171, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 61, + "latency_ms": 19422.107, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 22271.825, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 24122.253, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 81, + "latency_ms": 19316.069, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 16430.101, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 23924.628, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 140, + "latency_ms": 24029.01, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 22059.819, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 16934.516, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 18142.756, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 140, + "latency_ms": 25043.869, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 24961.061, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 21669.679, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 20961.564, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 39672.324, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 70597.795, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 93092.456, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 78973.886, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 22775.792, + "indexed_revision_matches": true + } + ], + "tier": "small", + "tier_code_files": 99, + "tier_provenance": "Stamped after the fact from a measured file count; the run predates tier stamping. Tier is a property of the repository, not of the run." +} diff --git a/benchmarks/context-retrieval/results/C-express.json b/benchmarks/context-retrieval/results/C-express.json new file mode 100644 index 00000000..d577a918 --- /dev/null +++ b/benchmarks/context-retrieval/results/C-express.json @@ -0,0 +1,6177 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "express", + "head": "a3714473feb3d2908add734d340e7755fd85e0a3" + }, + "tier": "small", + "tier_code_files": 141, + "corpus_counts": { + "cases": 28, + "path_leak": 9, + "baseline_blind": 19, + "multi_file": 14 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.7113, + "control": 0.2143, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "Fix hanging on large stack of sync routes", + "revision": "75e0c7a2c91665f44d053d83be15f8ecd0177f41", + "returned": [ + "test/req.range.js", + "test/acceptance/route-separation.js", + "test/res.append.js", + "test/req.accepts.js", + "lib/router/layer.js" + ] + }, + { + "provider_id": "random-files", + "query": "remove duplicate word (#6456)", + "revision": "5da5a11a498a3034623960861e542a3b39b00c94", + "returned": [ + "test/res.format.js", + "test/res.vary.js", + "test/res.cookie.js", + "lib/utils.js", + "examples/view-constructor/github-view.js" + ] + }, + { + "provider_id": "churn-ranked", + "query": "add Content-Length header only if Transfer-Encoding is not present (#4893)", + "revision": "59e205a57a04fced6bb7b8ec0b5dec29461a9996", + "returned": [ + "package.json", + ".github/workflows/ci.yml", + "lib/response.js", + ".github/workflows/legacy.yml", + ".github/workflows/scorecard.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix handling very large stacks of sync middleware", + "revision": "92c5ce59f51cce4b3598fd040117772fac42dce8", + "returned": [ + "package.json", + "appveyor.yml", + "lib/response.js", + "test/res.sendFile.js", + ".github/workflows/ci.yml" + ] + }, + { + "provider_id": "semble", + "query": "Remove `utils-merge` dependency - use spread syntax instead (#6091)", + "revision": "b11122be85377375465b2aa103da43301e56f128", + "returned": [ + "lib/application.js", + "lib/response.js", + "lib/express.js", + "lib/utils.js", + "lib/request.js" + ] + }, + { + "provider_id": "semble", + "query": "Fix handling very large stacks of sync middleware", + "revision": "92c5ce59f51cce4b3598fd040117772fac42dce8", + "returned": [ + "benchmarks/middleware.js", + "lib/express.js", + "lib/middleware/init.js", + "lib/application.js", + "lib/router/index.js" + ] + }, + { + "provider_id": "semble", + "query": "Fix incorrect middleware execution with unanchored RegExps", + "revision": "a1dbb113779c45a10f56a910de1bc5f7e750eaea", + "returned": [ + "lib/express.js", + "benchmarks/middleware.js", + "lib/middleware/init.js", + "lib/router/index.js", + "lib/middleware/query.js" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix issue where \"Request aborted\" may be logged in res.sendfile", + "revision": "5e9de5dcb6f1ff396038ded11951aa53c62a07a4", + "returned": [ + "lib/response.js", + "test/app.router.js" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix hanging on large stack of sync routes", + "revision": "75e0c7a2c91665f44d053d83be15f8ecd0177f41", + "returned": [ + "lib/router/index.js", + ".github/workflows/ci.yml" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix handling very large stacks of sync middleware", + "revision": "92c5ce59f51cce4b3598fd040117772fac42dce8", + "returned": [ + "lib/router/index.js" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "add Content-Length header only if Transfer-Encoding is not present (#4893)", + "revision": "59e205a57a04fced6bb7b8ec0b5dec29461a9996", + "returned": [ + "examples/ejs/views/footer.html", + "lib/response.js", + "History.md", + "test/fixtures/blog/index.html", + "examples/ejs/views/header.html" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "Fix issue where \"Request aborted\" may be logged in res.sendfile", + "revision": "5e9de5dcb6f1ff396038ded11951aa53c62a07a4", + "returned": [ + "History.md", + "examples/downloads/files/CCTV大赛上海分赛区.txt", + "lib/response.js", + "test/support/env.js" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "Fix hanging on large stack of sync routes", + "revision": "75e0c7a2c91665f44d053d83be15f8ecd0177f41", + "returned": [ + "examples/downloads/files/amazing.txt", + "package.json", + "test/fixtures/blog/index.html", + "examples/static-files/public/hello.txt", + "History.md" + ] + }, + { + "provider_id": "graphify", + "query": "Remove `utils-merge` dependency - use spread syntax instead (#6091)", + "revision": "b11122be85377375465b2aa103da43301e56f128", + "returned": [ + "package.json", + "lib/utils.js", + "examples/view-locals/user.js", + "lib/response.js", + "lib/application.js" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.0179, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0357, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0804, + "full_recall_at_20": 0.0357, + "mean_recall_at_1000_tokens": 0.0179, + "mean_recall_at_4000_tokens": 0.0179, + "mean_recall_at_16000_tokens": 0.0714, + "mean_precision_at_10": 0.0071, + "zero_hit_rate": 0.8571, + "median_tokens_delivered": 17557.5, + "median_latency_ms": 12.054, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 28, + "mean_recall_at_5": 0.0179, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0357, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0804, + "full_recall_at_20": 0.0357, + "mean_recall_at_1000_tokens": 0.0179, + "mean_recall_at_4000_tokens": 0.0179, + "mean_recall_at_16000_tokens": 0.0714, + "mean_precision_at_10": 0.0071, + "zero_hit_rate": 0.8571, + "median_tokens_delivered": 17557.5, + "median_latency_ms": 12.054, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 17126, + "median_latency_ms": 11.873, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.0263, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0526, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1184, + "full_recall_at_20": 0.0526, + "mean_recall_at_1000_tokens": 0.0263, + "mean_recall_at_4000_tokens": 0.0263, + "mean_recall_at_16000_tokens": 0.1053, + "mean_precision_at_10": 0.0105, + "zero_hit_rate": 0.7895, + "median_tokens_delivered": 17991, + "median_latency_ms": 12.479, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 28 + }, + "resources": { + "baseline_rss_mb": 15739.8, + "peak_rss_mb": 15764, + "delta_rss_mb": 24.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.4345, + "full_recall_at_5": 0.2857, + "mean_recall_at_10": 0.5625, + "full_recall_at_10": 0.3929, + "mean_recall_at_20": 0.619, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2143, + "mean_recall_at_16000_tokens": 0.4881, + "mean_precision_at_10": 0.0857, + "zero_hit_rate": 0.25, + "median_tokens_delivered": 55061.5, + "median_latency_ms": 21.6015, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 28, + "mean_recall_at_5": 0.4345, + "full_recall_at_5": 0.2857, + "mean_recall_at_10": 0.5625, + "full_recall_at_10": 0.3929, + "mean_recall_at_20": 0.619, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2143, + "mean_recall_at_16000_tokens": 0.4881, + "mean_precision_at_10": 0.0857, + "zero_hit_rate": 0.25, + "median_tokens_delivered": 55061.5, + "median_latency_ms": 21.6015, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.3148, + "full_recall_at_5": 0.1111, + "mean_recall_at_10": 0.3148, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.3148, + "full_recall_at_20": 0.1111, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.3148, + "mean_precision_at_10": 0.0556, + "zero_hit_rate": 0.4444, + "median_tokens_delivered": 54998, + "median_latency_ms": 21.685, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.4912, + "full_recall_at_5": 0.3684, + "mean_recall_at_10": 0.6798, + "full_recall_at_10": 0.5263, + "mean_recall_at_20": 0.7632, + "full_recall_at_20": 0.6842, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.3158, + "mean_recall_at_16000_tokens": 0.5702, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.1579, + "median_tokens_delivered": 55125, + "median_latency_ms": 21.579, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 28 + }, + "resources": { + "baseline_rss_mb": 15766.4, + "peak_rss_mb": 15779.7, + "delta_rss_mb": 13.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.5625, + "full_recall_at_5": 0.4643, + "mean_recall_at_10": 0.631, + "full_recall_at_10": 0.5714, + "mean_recall_at_20": 0.6607, + "full_recall_at_20": 0.6429, + "mean_recall_at_1000_tokens": 0.4643, + "mean_recall_at_4000_tokens": 0.6607, + "mean_recall_at_16000_tokens": 0.6607, + "mean_precision_at_10": 0.1143, + "zero_hit_rate": 0.3214, + "median_tokens_delivered": 3644.5, + "median_latency_ms": 782.83, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 28, + "mean_recall_at_5": 0.5625, + "full_recall_at_5": 0.4643, + "mean_recall_at_10": 0.631, + "full_recall_at_10": 0.5714, + "mean_recall_at_20": 0.6607, + "full_recall_at_20": 0.6429, + "mean_recall_at_1000_tokens": 0.4643, + "mean_recall_at_4000_tokens": 0.6607, + "mean_recall_at_16000_tokens": 0.6607, + "mean_precision_at_10": 0.1143, + "zero_hit_rate": 0.3214, + "median_tokens_delivered": 3644.5, + "median_latency_ms": 782.83, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.7593, + "full_recall_at_5": 0.6667, + "mean_recall_at_10": 0.8889, + "full_recall_at_10": 0.8889, + "mean_recall_at_20": 0.8889, + "full_recall_at_20": 0.8889, + "mean_recall_at_1000_tokens": 0.6481, + "mean_recall_at_4000_tokens": 0.8889, + "mean_recall_at_16000_tokens": 0.8889, + "mean_precision_at_10": 0.1667, + "zero_hit_rate": 0.1111, + "median_tokens_delivered": 3797, + "median_latency_ms": 776.077, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.4693, + "full_recall_at_5": 0.3684, + "mean_recall_at_10": 0.5088, + "full_recall_at_10": 0.4211, + "mean_recall_at_20": 0.5526, + "full_recall_at_20": 0.5263, + "mean_recall_at_1000_tokens": 0.3772, + "mean_recall_at_4000_tokens": 0.5526, + "mean_recall_at_16000_tokens": 0.5526, + "mean_precision_at_10": 0.0895, + "zero_hit_rate": 0.4211, + "median_tokens_delivered": 3591, + "median_latency_ms": 801.052, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 28 + }, + "resources": { + "baseline_rss_mb": 15795.2, + "peak_rss_mb": 17745.6, + "delta_rss_mb": 1950.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "jcodemunch", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.1161, + "full_recall_at_5": 0.0714, + "mean_recall_at_10": 0.1161, + "full_recall_at_10": 0.0714, + "mean_recall_at_20": 0.1161, + "full_recall_at_20": 0.0714, + "mean_recall_at_1000_tokens": 0.1161, + "mean_recall_at_4000_tokens": 0.1161, + "mean_recall_at_16000_tokens": 0.1161, + "mean_precision_at_10": 0.125, + "zero_hit_rate": 0.8214, + "median_tokens_delivered": 350, + "median_latency_ms": 390.5705, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 7 + }, + "baseline_missed": { + "cases": 28, + "mean_recall_at_5": 0.1161, + "full_recall_at_5": 0.0714, + "mean_recall_at_10": 0.1161, + "full_recall_at_10": 0.0714, + "mean_recall_at_20": 0.1161, + "full_recall_at_20": 0.0714, + "mean_recall_at_1000_tokens": 0.1161, + "mean_recall_at_4000_tokens": 0.1161, + "mean_recall_at_16000_tokens": 0.1161, + "mean_precision_at_10": 0.125, + "zero_hit_rate": 0.8214, + "median_tokens_delivered": 350, + "median_latency_ms": 390.5705, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 7 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.1111, + "full_recall_at_5": 0.1111, + "mean_recall_at_10": 0.1111, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.1111, + "full_recall_at_20": 0.1111, + "mean_recall_at_1000_tokens": 0.1111, + "mean_recall_at_4000_tokens": 0.1111, + "mean_recall_at_16000_tokens": 0.1111, + "mean_precision_at_10": 0.1111, + "zero_hit_rate": 0.8889, + "median_tokens_delivered": 350, + "median_latency_ms": 375.21, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 1 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.1184, + "full_recall_at_5": 0.0526, + "mean_recall_at_10": 0.1184, + "full_recall_at_10": 0.0526, + "mean_recall_at_20": 0.1184, + "full_recall_at_20": 0.0526, + "mean_recall_at_1000_tokens": 0.1184, + "mean_recall_at_4000_tokens": 0.1184, + "mean_recall_at_16000_tokens": 0.1184, + "mean_precision_at_10": 0.1316, + "zero_hit_rate": 0.7895, + "median_tokens_delivered": 329, + "median_latency_ms": 392.352, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 6 + } + }, + "outcomes": { + "indexed-but-returned-nothing": 7, + "answered": 21 + }, + "resources": { + "baseline_rss_mb": 17254.5, + "peak_rss_mb": 18625.2, + "delta_rss_mb": 1370.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "cocoindex-code", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.1786, + "full_recall_at_5": 0.1429, + "mean_recall_at_10": 0.2321, + "full_recall_at_10": 0.1786, + "mean_recall_at_20": 0.2321, + "full_recall_at_20": 0.1786, + "mean_recall_at_1000_tokens": 0.0893, + "mean_recall_at_4000_tokens": 0.2321, + "mean_recall_at_16000_tokens": 0.2321, + "mean_precision_at_10": 0.0518, + "zero_hit_rate": 0.7143, + "median_tokens_delivered": 2607, + "median_latency_ms": 922.8415, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 28, + "mean_recall_at_5": 0.1786, + "full_recall_at_5": 0.1429, + "mean_recall_at_10": 0.2321, + "full_recall_at_10": 0.1786, + "mean_recall_at_20": 0.2321, + "full_recall_at_20": 0.1786, + "mean_recall_at_1000_tokens": 0.0893, + "mean_recall_at_4000_tokens": 0.2321, + "mean_recall_at_16000_tokens": 0.2321, + "mean_precision_at_10": 0.0518, + "zero_hit_rate": 0.7143, + "median_tokens_delivered": 2607, + "median_latency_ms": 922.8415, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.1111, + "full_recall_at_5": 0.1111, + "mean_recall_at_10": 0.2222, + "full_recall_at_10": 0.2222, + "mean_recall_at_20": 0.2222, + "full_recall_at_20": 0.2222, + "mean_recall_at_1000_tokens": 0.0556, + "mean_recall_at_4000_tokens": 0.2222, + "mean_recall_at_16000_tokens": 0.2222, + "mean_precision_at_10": 0.0593, + "zero_hit_rate": 0.7778, + "median_tokens_delivered": 2607, + "median_latency_ms": 834.263, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.2105, + "full_recall_at_5": 0.1579, + "mean_recall_at_10": 0.2368, + "full_recall_at_10": 0.1579, + "mean_recall_at_20": 0.2368, + "full_recall_at_20": 0.1579, + "mean_recall_at_1000_tokens": 0.1053, + "mean_recall_at_4000_tokens": 0.2368, + "mean_recall_at_16000_tokens": 0.2368, + "mean_precision_at_10": 0.0482, + "zero_hit_rate": 0.6842, + "median_tokens_delivered": 2633, + "median_latency_ms": 1187.397, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 28 + }, + "resources": { + "baseline_rss_mb": 17949, + "peak_rss_mb": 25538.8, + "delta_rss_mb": 7589.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.5149, + "full_recall_at_5": 0.3929, + "mean_recall_at_10": 0.6042, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.6399, + "full_recall_at_20": 0.5357, + "mean_recall_at_1000_tokens": 0.5089, + "mean_recall_at_4000_tokens": 0.5685, + "mean_recall_at_16000_tokens": 0.6399, + "mean_precision_at_10": 0.1015, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 1681.5, + "median_latency_ms": 1060.489, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 28, + "mean_recall_at_5": 0.5149, + "full_recall_at_5": 0.3929, + "mean_recall_at_10": 0.6042, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.6399, + "full_recall_at_20": 0.5357, + "mean_recall_at_1000_tokens": 0.5089, + "mean_recall_at_4000_tokens": 0.5685, + "mean_recall_at_16000_tokens": 0.6399, + "mean_precision_at_10": 0.1015, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 1681.5, + "median_latency_ms": 1060.489, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.6667, + "full_recall_at_5": 0.5556, + "mean_recall_at_10": 0.6667, + "full_recall_at_10": 0.5556, + "mean_recall_at_20": 0.6667, + "full_recall_at_20": 0.5556, + "mean_recall_at_1000_tokens": 0.6667, + "mean_recall_at_4000_tokens": 0.6667, + "mean_recall_at_16000_tokens": 0.6667, + "mean_precision_at_10": 0.1159, + "zero_hit_rate": 0.2222, + "median_tokens_delivered": 1676, + "median_latency_ms": 1040.977, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.443, + "full_recall_at_5": 0.3158, + "mean_recall_at_10": 0.5746, + "full_recall_at_10": 0.4737, + "mean_recall_at_20": 0.6272, + "full_recall_at_20": 0.5263, + "mean_recall_at_1000_tokens": 0.4342, + "mean_recall_at_4000_tokens": 0.5219, + "mean_recall_at_16000_tokens": 0.6272, + "mean_precision_at_10": 0.0947, + "zero_hit_rate": 0.3158, + "median_tokens_delivered": 1687, + "median_latency_ms": 1063, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 27, + "indexed-but-returned-nothing": 1 + }, + "resources": { + "baseline_rss_mb": 20384.2, + "peak_rss_mb": 21803.7, + "delta_rss_mb": 1419.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 9977856 + } + }, + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.5357, + "full_recall_at_5": 0.3929, + "mean_recall_at_10": 0.6131, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.7113, + "full_recall_at_20": 0.6071, + "mean_recall_at_1000_tokens": 0.7113, + "mean_recall_at_4000_tokens": 0.7113, + "mean_recall_at_16000_tokens": 0.7113, + "mean_precision_at_10": 0.1027, + "zero_hit_rate": 0.1786, + "median_tokens_delivered": 700.5, + "median_latency_ms": 245.949, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 28, + "mean_recall_at_5": 0.5357, + "full_recall_at_5": 0.3929, + "mean_recall_at_10": 0.6131, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.7113, + "full_recall_at_20": 0.6071, + "mean_recall_at_1000_tokens": 0.7113, + "mean_recall_at_4000_tokens": 0.7113, + "mean_recall_at_16000_tokens": 0.7113, + "mean_precision_at_10": 0.1027, + "zero_hit_rate": 0.1786, + "median_tokens_delivered": 700.5, + "median_latency_ms": 245.949, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.5741, + "full_recall_at_5": 0.3333, + "mean_recall_at_10": 0.7407, + "full_recall_at_10": 0.5556, + "mean_recall_at_20": 0.7407, + "full_recall_at_20": 0.5556, + "mean_recall_at_1000_tokens": 0.7407, + "mean_recall_at_4000_tokens": 0.7407, + "mean_recall_at_16000_tokens": 0.7407, + "mean_precision_at_10": 0.1222, + "zero_hit_rate": 0.1111, + "median_tokens_delivered": 651, + "median_latency_ms": 245.881, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.5175, + "full_recall_at_5": 0.4211, + "mean_recall_at_10": 0.5526, + "full_recall_at_10": 0.4737, + "mean_recall_at_20": 0.6974, + "full_recall_at_20": 0.6316, + "mean_recall_at_1000_tokens": 0.6974, + "mean_recall_at_4000_tokens": 0.6974, + "mean_recall_at_16000_tokens": 0.6974, + "mean_precision_at_10": 0.0935, + "zero_hit_rate": 0.2105, + "median_tokens_delivered": 722, + "median_latency_ms": 246.017, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 28 + }, + "resources": { + "baseline_rss_mb": 20141.3, + "peak_rss_mb": 20889.2, + "delta_rss_mb": 747.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 197193728 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15887, + "latency_ms": 18.394, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.send.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24167, + "latency_ms": 12.639, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [], + "missed": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17991, + "latency_ms": 12.587, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15115, + "latency_ms": 12.62, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8352, + "latency_ms": 12.063, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25943, + "latency_ms": 12.844, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24105, + "latency_ms": 12.664, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.redirect.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17989, + "latency_ms": 12.584, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.cookie.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10199, + "latency_ms": 11.849, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25824, + "latency_ms": 11.873, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [ + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 21567, + "latency_ms": 12.002, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14979, + "latency_ms": 11.581, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21348, + "latency_ms": 11.17, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28836, + "latency_ms": 12.137, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "test/Route.js" + ], + "missed": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 25626, + "latency_ms": 14.651, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26880, + "latency_ms": 14.078, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12752, + "latency_ms": 11.995, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12150, + "latency_ms": 11.715, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [], + "missed": [ + "lib/router/index.js", + "test/app.router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12313, + "latency_ms": 12.91, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14066, + "latency_ms": 11.722, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24966, + "latency_ms": 12.045, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [], + "missed": [ + "lib/response.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18218, + "latency_ms": 12.479, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 22036, + "latency_ms": 11.723, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [], + "missed": [ + "examples/search/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17126, + "latency_ms": 12.072, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15153, + "latency_ms": 11.095, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [], + "missed": [ + "lib/router/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15550, + "latency_ms": 11.78, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [ + "test/res.format.js" + ], + "missed": [ + "test/app.router.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 15695, + "latency_ms": 11.32, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10590, + "latency_ms": 11.133, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 48438, + "latency_ms": 24.396, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [ + "lib/response.js", + "test/res.send.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 63857, + "latency_ms": 22.846, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 68922, + "latency_ms": 22.822, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 46929, + "latency_ms": 27.519, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 44158, + "latency_ms": 21.789, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 69122, + "latency_ms": 21.07, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48438, + "latency_ms": 22.549, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.redirect.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 56208, + "latency_ms": 21.685, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.cookie.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 54109, + "latency_ms": 20.1, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69044, + "latency_ms": 19.301, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 57263, + "latency_ms": 21.696, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 48439, + "latency_ms": 20.622, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68691, + "latency_ms": 22.241, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68940, + "latency_ms": 21.624, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 57070, + "latency_ms": 19.649, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [ + "test/Router.js" + ], + "missed": [ + "lib/router/route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 55125, + "latency_ms": 21.579, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54998, + "latency_ms": 21.846, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 63660, + "latency_ms": 20.518, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [ + "lib/router/index.js", + "test/app.router.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 46930, + "latency_ms": 20.784, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63835, + "latency_ms": 21.079, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.jsonp.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 47132, + "latency_ms": 22.071, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 46929, + "latency_ms": 20.67, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 52216, + "latency_ms": 22.581, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [], + "missed": [ + "examples/search/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63752, + "latency_ms": 22.93, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.json.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 47164, + "latency_ms": 21.067, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 44146, + "latency_ms": 19.515, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [ + "test/app.router.js" + ], + "missed": [ + "test/res.format.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 69122, + "latency_ms": 21.319, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 52218, + "latency_ms": 21.01, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3250, + "latency_ms": 870.965, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [ + "lib/response.js", + "test/res.send.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3324, + "latency_ms": 577.086, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3130, + "latency_ms": 589.526, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3845, + "latency_ms": 595.943, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4209, + "latency_ms": 552.372, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3947, + "latency_ms": 604.348, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [ + "test/res.type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3506, + "latency_ms": 595.198, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [ + "lib/response.js", + "test/res.redirect.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3797, + "latency_ms": 649.632, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3642, + "latency_ms": 619.412, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3376, + "latency_ms": 625.864, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3486, + "latency_ms": 643.895, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3591, + "latency_ms": 614.371, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4165, + "latency_ms": 644.462, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3765, + "latency_ms": 780.468, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "lib/router/route.js" + ], + "missed": [ + "test/Route.js", + "test/Router.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 3608, + "latency_ms": 805.668, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [ + "lib/router/route.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3454, + "latency_ms": 816.514, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [ + "lib/router/route.js", + "test/Route.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3673, + "latency_ms": 888.686, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3841, + "latency_ms": 801.052, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [ + "lib/router/index.js", + "test/app.router.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3590, + "latency_ms": 884.012, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3661, + "latency_ms": 845.61, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3911, + "latency_ms": 785.192, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3640, + "latency_ms": 836.859, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3744, + "latency_ms": 830.808, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [ + "examples/search/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3937, + "latency_ms": 831.36, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3963, + "latency_ms": 908.686, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3647, + "latency_ms": 776.077, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [], + "missed": [ + "test/app.router.js", + "test/res.format.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3399, + "latency_ms": 940.148, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3189, + "latency_ms": 908.316, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 97, + "latency_ms": 406.16, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.send.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 436, + "latency_ms": 353.204, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [], + "missed": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 454, + "latency_ms": 338.802, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 329, + "latency_ms": 385.225, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.sendFile.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 315, + "latency_ms": 423.344, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48, + "latency_ms": 492.967, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 254, + "latency_ms": 879.729, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.redirect.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 442, + "latency_ms": 396.655, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.cookie.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 350, + "latency_ms": 375.21, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 342, + "latency_ms": 431.753, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [ + "test/Router.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 387, + "latency_ms": 384, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 59, + "latency_ms": 450.202, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 256, + "latency_ms": 363.12, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 461, + "latency_ms": 388.789, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [ + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 1, + "recall_at_10": 0.25, + "precision_at_10": 1, + "recall_at_20": 0.25, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 419, + "latency_ms": 359.447, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 280, + "latency_ms": 420.889, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 369, + "latency_ms": 371.963, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 454, + "latency_ms": 418.441, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [], + "missed": [ + "lib/router/index.js", + "test/app.router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 435, + "latency_ms": 359.301, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 322, + "latency_ms": 377.362, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 350, + "latency_ms": 374.919, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [], + "missed": [ + "lib/response.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 395, + "latency_ms": 348.514, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48, + "latency_ms": 452.847, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [], + "missed": [ + "examples/search/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 358, + "latency_ms": 414.252, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 443, + "latency_ms": 354.923, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 339, + "latency_ms": 445.081, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [], + "missed": [ + "test/app.router.js", + "test/res.format.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 94, + "latency_ms": 449.819, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 392, + "latency_ms": 392.352, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2986, + "latency_ms": 3272.523, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.send.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4352, + "latency_ms": 10823.181, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [], + "missed": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3479, + "latency_ms": 64386.776, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2633, + "latency_ms": 129267.416, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.sendFile.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 4111, + "latency_ms": 129823.887, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3022, + "latency_ms": 26728.114, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2753, + "latency_ms": 16502.439, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [ + "lib/response.js", + "test/res.redirect.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 2929, + "latency_ms": 28871.76, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4901, + "latency_ms": 3685.27, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3170, + "latency_ms": 115279.993, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "test/Router.js" + ], + "missed": [ + "lib/router/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 3116, + "latency_ms": 12608.054, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4025, + "latency_ms": 151870.359, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 828.107, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 855.015, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [], + "missed": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 947.428, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 898.255, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 834.263, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1187.397, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [], + "missed": [ + "lib/router/index.js", + "test/app.router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 803.397, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 654.152, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 785.451, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [], + "missed": [ + "lib/response.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 707.268, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 743.192, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [], + "missed": [ + "examples/search/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 740.273, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 857.18, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [], + "missed": [ + "lib/router/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 831.304, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [], + "missed": [ + "test/app.router.js", + "test/res.format.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3264, + "latency_ms": 820.518, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 726.492, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1150, + "latency_ms": 1107.786, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.send.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2445, + "latency_ms": 1122.143, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/application.js", + "lib/response.js" + ], + "missed": [ + "test/res.cookie.js" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 5687, + "latency_ms": 1042.812, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 8068, + "latency_ms": 1029.523, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1687, + "latency_ms": 924.404, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1674, + "latency_ms": 1032.962, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3297, + "latency_ms": 1036.459, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [ + "test/res.redirect.js" + ], + "missed": [ + "lib/response.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2483, + "latency_ms": 1098.65, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1676, + "latency_ms": 1006.982, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 878, + "latency_ms": 1166.939, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 5239, + "latency_ms": 1069.832, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2710, + "latency_ms": 1143.638, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1370, + "latency_ms": 988.117, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 806, + "latency_ms": 1063, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Router.js" + ], + "missed": [ + "test/Route.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 3, + "tokens_delivered": 5296, + "latency_ms": 1018.545, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [ + "lib/router/route.js" + ], + "missed": [ + "test/Router.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1466, + "latency_ms": 1057.978, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [ + "lib/router/route.js" + ], + "missed": [ + "test/Route.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2044, + "latency_ms": 1142.223, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1242, + "latency_ms": 1066.084, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [ + "lib/router/index.js", + "test/app.router.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 4647, + "latency_ms": 1101.449, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7, + "latency_ms": 1013.558, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1673, + "latency_ms": 1003.259, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1675, + "latency_ms": 1146.015, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1694, + "latency_ms": 1027.475, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [ + "examples/search/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1658, + "latency_ms": 1040.977, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3437, + "latency_ms": 1029.889, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4086, + "latency_ms": 1340.726, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [], + "missed": [ + "test/app.router.js", + "test/res.format.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1539, + "latency_ms": 1151.59, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1174, + "latency_ms": 1126.218, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1598, + "latency_ms": 880.365, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [ + "test/res.send.js" + ], + "missed": [ + "lib/response.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1485, + "latency_ms": 227.044, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1211, + "latency_ms": 228.554, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1602, + "latency_ms": 221.189, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 823, + "latency_ms": 202.192, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 615, + "latency_ms": 341.574, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 206, + "latency_ms": 246.322, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.redirect.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 650, + "latency_ms": 254.079, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "test/res.cookie.js" + ], + "missed": [ + "lib/response.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1132, + "latency_ms": 245.881, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [ + "examples/auth/views/head.ejs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 651, + "latency_ms": 230.026, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 772, + "latency_ms": 246.017, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 49, + "latency_ms": 329.919, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 698, + "latency_ms": 238.796, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1044, + "latency_ms": 250.317, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "test/Router.js" + ], + "missed": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 19, + "tokens_delivered": 703, + "latency_ms": 264.994, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [ + "lib/router/route.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 722, + "latency_ms": 245.721, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [ + "lib/router/route.js", + "test/Route.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 859, + "latency_ms": 319.965, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 104, + "latency_ms": 263.372, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [ + "test/app.router.js" + ], + "missed": [ + "lib/router/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 671, + "latency_ms": 238.092, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 439, + "latency_ms": 230.935, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [ + "test/res.jsonp.js" + ], + "missed": [ + "lib/response.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 770, + "latency_ms": 265.392, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 767, + "latency_ms": 264.839, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 592, + "latency_ms": 335.496, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [ + "examples/search/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 616, + "latency_ms": 254.201, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [ + "test/res.json.js", + "test/res.jsonp.js" + ], + "missed": [ + "lib/response.js" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 587, + "latency_ms": 233.438, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 546, + "latency_ms": 207.427, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [], + "missed": [ + "test/app.router.js", + "test/res.format.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36, + "latency_ms": 245.546, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 924, + "latency_ms": 243.85, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/C-flask.json b/benchmarks/context-retrieval/results/C-flask.json new file mode 100644 index 00000000..28703a0e --- /dev/null +++ b/benchmarks/context-retrieval/results/C-flask.json @@ -0,0 +1,8290 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "flask", + "head": "d318b683471101618febed18996405ad26462110" + }, + "tier": "small", + "tier_code_files": 83, + "corpus_counts": { + "cases": 39, + "path_leak": 7, + "baseline_blind": 32, + "multi_file": 17 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.5278, + "control": 0.0321, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "fix uninstalled package tests under tox", + "revision": "88bcf78439b66b5fcba9f4d3a1c56307f98dbf1d", + "returned": [ + "examples/tutorial/tests/test_factory.py", + "tests/test_apps/helloworld/wsgi.py", + "tests/test_apps/subdomaintestmodule/__init__.py", + "tests/test_reqctx.py", + "docs/conf.py" + ] + }, + { + "provider_id": "random-files", + "query": "fix subdomain_matching=False behavior", + "revision": "07c7d5730a2685ef2281cc635e289685e5c3d478", + "returned": [ + ".readthedocs.yaml", + "tests/test_views.py", + "src/flask/cli.py", + "tests/test_apps/cliapp/inner1/__init__.py", + "src/flask/config.py" + ] + }, + { + "provider_id": "random-files", + "query": "fix provide_automatic_options override", + "revision": "d3b78fd18a8d9e224cb9ef58a23cec9b1ffc9ce9", + "returned": [ + "tests/test_async.py", + "tests/test_views.py", + "tests/conftest.py", + "tests/test_apps/cliapp/app.py", + "tests/type_check/typing_app_decorators.py" + ] + }, + { + "provider_id": "churn-ranked", + "query": "fix importing Markup from flask", + "revision": "345f18442ceb29f9e365af99fc85bdc5986323d2", + "returned": [ + "src/flask/app.py", + ".pre-commit-config.yaml", + "src/flask/helpers.py", + "src/flask/blueprints.py", + "src/flask/__init__.py" + ] + }, + { + "provider_id": "churn-ranked", + "query": "fix pytest 7 warnings", + "revision": "6f7d99ce4b7dc2bad52683e64fec7008dd0b580f", + "returned": [ + "src/flask/app.py", + "src/flask/helpers.py", + "src/flask/blueprints.py", + "src/flask/cli.py", + "tests/test_basic.py" + ] + }, + { + "provider_id": "churn-ranked", + "query": "fix annotation for json.loads", + "revision": "69f71b4d94006678a2463f76a4504208d195c367", + "returned": [ + "src/flask/app.py", + "src/flask/helpers.py", + ".pre-commit-config.yaml", + "src/flask/blueprints.py", + "src/flask/cli.py" + ] + }, + { + "provider_id": "semble", + "query": "fix mypy findings", + "revision": "2d31dce8262a68875c177645fa0c7585a45ac447", + "returned": [ + "src/flask/typing.py", + "src/flask/cli.py", + "src/flask/helpers.py", + "src/flask/sansio/scaffold.py", + "src/flask/debughelpers.py" + ] + }, + { + "provider_id": "semble", + "query": "fix type hint for `cli_runner.invoke`", + "revision": "ea08f155d8d37d515559534bbe0f80ef5185466c", + "returned": [ + "src/flask/cli.py", + "src/flask/testing.py", + "src/flask/app.py", + "src/flask/sansio/app.py", + "src/flask/sansio/blueprints.py" + ] + }, + { + "provider_id": "semble", + "query": "fix annotation for json.loads", + "revision": "69f71b4d94006678a2463f76a4504208d195c367", + "returned": [ + "src/flask/json/tag.py", + "src/flask/scaffold.py", + "src/flask/config.py", + "src/flask/app.py", + "src/flask/blueprints.py" + ] + }, + { + "provider_id": "jcodemunch", + "query": "fix flake8 bugbear errors", + "revision": "8f13f5b6d672f1ba434f9c8ebe2c1b1dd385962e", + "returned": [ + "tests/test_basic.py", + "examples/tutorial/tests/test_auth.py" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix IPv6 server name parsing", + "revision": "de8429ffda8cfb54db175529e2bae72a24e1fa7e", + "returned": [ + "tests/test_basic.py" + ] + }, + { + "provider_id": "jcodemunch", + "query": "fix annotation for json.loads", + "revision": "69f71b4d94006678a2463f76a4504208d195c367", + "returned": [ + "src/flask/json/__init__.py", + ".pre-commit-config.yaml" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "fix importing Markup from flask", + "revision": "345f18442ceb29f9e365af99fc85bdc5986323d2", + "returned": [ + "tests/test_apps/cliapp/app.py", + "tests/test_apps/cliapp/importerrorapp.py", + "tests/test_apps/cliapp/inner1/inner2/flask.py", + "tests/test_apps/cliapp/inner1/__init__.py", + "tests/test_apps/subdomaintestmodule/__init__.py" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "fix super call in list comprehension", + "revision": "3435d2ff1589eb0c1a85cc294a20985910a1a606", + "returned": [ + "tests/templates/nested/nested.txt", + "tests/test_apps/blueprintapp/apps/admin/static/css/test.css", + "tests/test_basic.py", + "tests/test_apps/blueprintapp/apps/frontend/templates/frontend/index.html", + "tests/templates/template_filter.html" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.0427, + "full_recall_at_5": 0.0256, + "mean_recall_at_10": 0.0962, + "full_recall_at_10": 0.0513, + "mean_recall_at_20": 0.1859, + "full_recall_at_20": 0.1026, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0171, + "mean_recall_at_16000_tokens": 0.094, + "mean_precision_at_10": 0.0179, + "zero_hit_rate": 0.6667, + "median_tokens_delivered": 30503, + "median_latency_ms": 11.854, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 37, + "mean_recall_at_5": 0.018, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0473, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1419, + "full_recall_at_20": 0.0541, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.018, + "mean_recall_at_16000_tokens": 0.045, + "mean_precision_at_10": 0.0135, + "zero_hit_rate": 0.7027, + "median_tokens_delivered": 30706, + "median_latency_ms": 11.917, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 26740, + "median_latency_ms": 11.5975, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0476, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.119, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0714, + "mean_precision_at_10": 0.0143, + "zero_hit_rate": 0.7143, + "median_tokens_delivered": 26267, + "median_latency_ms": 12.291, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.0521, + "full_recall_at_5": 0.0313, + "mean_recall_at_10": 0.1068, + "full_recall_at_10": 0.0625, + "mean_recall_at_20": 0.2005, + "full_recall_at_20": 0.125, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0208, + "mean_recall_at_16000_tokens": 0.099, + "mean_precision_at_10": 0.0188, + "zero_hit_rate": 0.6563, + "median_tokens_delivered": 31713, + "median_latency_ms": 11.843, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 39 + }, + "resources": { + "baseline_rss_mb": 15739.8, + "peak_rss_mb": 15762.6, + "delta_rss_mb": 22.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.2671, + "full_recall_at_5": 0.1795, + "mean_recall_at_10": 0.4573, + "full_recall_at_10": 0.3077, + "mean_recall_at_20": 0.6026, + "full_recall_at_20": 0.4359, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0321, + "mean_recall_at_16000_tokens": 0.1603, + "mean_precision_at_10": 0.0718, + "zero_hit_rate": 0.2308, + "median_tokens_delivered": 80328, + "median_latency_ms": 24.474, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 37, + "mean_recall_at_5": 0.2545, + "full_recall_at_5": 0.1622, + "mean_recall_at_10": 0.455, + "full_recall_at_10": 0.2973, + "mean_recall_at_20": 0.6081, + "full_recall_at_20": 0.4324, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0338, + "mean_recall_at_16000_tokens": 0.1419, + "mean_precision_at_10": 0.073, + "zero_hit_rate": 0.2162, + "median_tokens_delivered": 80328, + "median_latency_ms": 24.474, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.5, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.5, + "mean_precision_at_10": 0.05, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 89733, + "median_latency_ms": 24.6155, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0.1429, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.4048, + "full_recall_at_10": 0.1429, + "mean_recall_at_20": 0.4762, + "full_recall_at_20": 0.2857, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0714, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 98043, + "median_latency_ms": 26.065, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.2943, + "full_recall_at_5": 0.2188, + "mean_recall_at_10": 0.4688, + "full_recall_at_10": 0.3438, + "mean_recall_at_20": 0.6302, + "full_recall_at_20": 0.4688, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0391, + "mean_recall_at_16000_tokens": 0.1953, + "mean_precision_at_10": 0.0719, + "zero_hit_rate": 0.2188, + "median_tokens_delivered": 80264.5, + "median_latency_ms": 24.45, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 39 + }, + "resources": { + "baseline_rss_mb": 15777, + "peak_rss_mb": 15857.2, + "delta_rss_mb": 80.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.3355, + "full_recall_at_5": 0.1795, + "mean_recall_at_10": 0.4594, + "full_recall_at_10": 0.2821, + "mean_recall_at_20": 0.6303, + "full_recall_at_20": 0.4615, + "mean_recall_at_1000_tokens": 0.3013, + "mean_recall_at_4000_tokens": 0.5278, + "mean_recall_at_16000_tokens": 0.6303, + "mean_precision_at_10": 0.0744, + "zero_hit_rate": 0.2051, + "median_tokens_delivered": 4174, + "median_latency_ms": 823.585, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 37, + "mean_recall_at_5": 0.3536, + "full_recall_at_5": 0.1892, + "mean_recall_at_10": 0.4842, + "full_recall_at_10": 0.2973, + "mean_recall_at_20": 0.6374, + "full_recall_at_20": 0.4595, + "mean_recall_at_1000_tokens": 0.3176, + "mean_recall_at_4000_tokens": 0.5563, + "mean_recall_at_16000_tokens": 0.6374, + "mean_precision_at_10": 0.0784, + "zero_hit_rate": 0.1892, + "median_tokens_delivered": 4174, + "median_latency_ms": 810.485, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.5, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.5, + "mean_precision_at_10": 0, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 4070, + "median_latency_ms": 871.299, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0.3095, + "full_recall_at_5": 0.1429, + "mean_recall_at_10": 0.4524, + "full_recall_at_10": 0.2857, + "mean_recall_at_20": 0.5238, + "full_recall_at_20": 0.2857, + "mean_recall_at_1000_tokens": 0.119, + "mean_recall_at_4000_tokens": 0.5238, + "mean_recall_at_16000_tokens": 0.5238, + "mean_precision_at_10": 0.0857, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 4117, + "median_latency_ms": 810.485, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.3411, + "full_recall_at_5": 0.1875, + "mean_recall_at_10": 0.4609, + "full_recall_at_10": 0.2813, + "mean_recall_at_20": 0.6536, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0.3411, + "mean_recall_at_4000_tokens": 0.5286, + "mean_recall_at_16000_tokens": 0.6536, + "mean_precision_at_10": 0.0719, + "zero_hit_rate": 0.1875, + "median_tokens_delivered": 4179.5, + "median_latency_ms": 831.542, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 39 + }, + "resources": { + "baseline_rss_mb": 15915.4, + "peak_rss_mb": 17921.4, + "delta_rss_mb": 2006, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "jcodemunch", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.1944, + "full_recall_at_5": 0.0769, + "mean_recall_at_10": 0.1944, + "full_recall_at_10": 0.0769, + "mean_recall_at_20": 0.1944, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0.1944, + "mean_recall_at_4000_tokens": 0.1944, + "mean_recall_at_16000_tokens": 0.1944, + "mean_precision_at_10": 0.1987, + "zero_hit_rate": 0.641, + "median_tokens_delivered": 434, + "median_latency_ms": 507.097, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 37, + "mean_recall_at_5": 0.1779, + "full_recall_at_5": 0.0541, + "mean_recall_at_10": 0.1779, + "full_recall_at_10": 0.0541, + "mean_recall_at_20": 0.1779, + "full_recall_at_20": 0.0541, + "mean_recall_at_1000_tokens": 0.1779, + "mean_recall_at_4000_tokens": 0.1779, + "mean_recall_at_16000_tokens": 0.1779, + "mean_precision_at_10": 0.1959, + "zero_hit_rate": 0.6486, + "median_tokens_delivered": 437, + "median_latency_ms": 507.299, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.5, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0.5, + "mean_recall_at_4000_tokens": 0.5, + "mean_recall_at_16000_tokens": 0.5, + "mean_precision_at_10": 0.25, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 198.5, + "median_latency_ms": 493.686, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0.1905, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1905, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1905, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.1905, + "mean_recall_at_4000_tokens": 0.1905, + "mean_recall_at_16000_tokens": 0.1905, + "mean_precision_at_10": 0.1905, + "zero_hit_rate": 0.5714, + "median_tokens_delivered": 453, + "median_latency_ms": 564.02, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.1953, + "full_recall_at_5": 0.0938, + "mean_recall_at_10": 0.1953, + "full_recall_at_10": 0.0938, + "mean_recall_at_20": 0.1953, + "full_recall_at_20": 0.0938, + "mean_recall_at_1000_tokens": 0.1953, + "mean_recall_at_4000_tokens": 0.1953, + "mean_recall_at_16000_tokens": 0.1953, + "mean_precision_at_10": 0.2005, + "zero_hit_rate": 0.6563, + "median_tokens_delivered": 406.5, + "median_latency_ms": 500.611, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 39 + }, + "resources": { + "baseline_rss_mb": 17085.6, + "peak_rss_mb": 20168.8, + "delta_rss_mb": 3083.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "cocoindex-code", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.0256, + "full_recall_at_5": 0.0256, + "mean_recall_at_10": 0.1068, + "full_recall_at_10": 0.0769, + "mean_recall_at_20": 0.1282, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0.0598, + "mean_recall_at_4000_tokens": 0.1282, + "mean_recall_at_16000_tokens": 0.1282, + "mean_precision_at_10": 0.0154, + "zero_hit_rate": 0.8205, + "median_tokens_delivered": 2607, + "median_latency_ms": 785.762, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 1 + }, + "baseline_missed": { + "cases": 37, + "mean_recall_at_5": 0.027, + "full_recall_at_5": 0.027, + "mean_recall_at_10": 0.1126, + "full_recall_at_10": 0.0811, + "mean_recall_at_20": 0.1351, + "full_recall_at_20": 0.0811, + "mean_recall_at_1000_tokens": 0.0631, + "mean_recall_at_4000_tokens": 0.1351, + "mean_recall_at_16000_tokens": 0.1351, + "mean_precision_at_10": 0.0162, + "zero_hit_rate": 0.8108, + "median_tokens_delivered": 2607, + "median_latency_ms": 787.466, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 1 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 2607, + "median_latency_ms": 669.0835, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.119, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.119, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.0476, + "mean_recall_at_4000_tokens": 0.119, + "mean_recall_at_16000_tokens": 0.119, + "mean_precision_at_10": 0.0286, + "zero_hit_rate": 0.7143, + "median_tokens_delivered": 2607, + "median_latency_ms": 852.278, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.0313, + "full_recall_at_5": 0.0313, + "mean_recall_at_10": 0.1042, + "full_recall_at_10": 0.0938, + "mean_recall_at_20": 0.1302, + "full_recall_at_20": 0.0938, + "mean_recall_at_1000_tokens": 0.0625, + "mean_recall_at_4000_tokens": 0.1302, + "mean_recall_at_16000_tokens": 0.1302, + "mean_precision_at_10": 0.0125, + "zero_hit_rate": 0.8438, + "median_tokens_delivered": 2607, + "median_latency_ms": 782.773, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 1 + } + }, + "outcomes": { + "answered": 38, + "unavailable-at-query-time": 1 + }, + "resources": { + "baseline_rss_mb": 18516.2, + "peak_rss_mb": 25561, + "delta_rss_mb": 7044.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.2714, + "full_recall_at_5": 0.1026, + "mean_recall_at_10": 0.312, + "full_recall_at_10": 0.1282, + "mean_recall_at_20": 0.3547, + "full_recall_at_20": 0.1795, + "mean_recall_at_1000_tokens": 0.2863, + "mean_recall_at_4000_tokens": 0.3547, + "mean_recall_at_16000_tokens": 0.3547, + "mean_precision_at_10": 0.0765, + "zero_hit_rate": 0.4872, + "median_tokens_delivered": 1704, + "median_latency_ms": 1271.991, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 37, + "mean_recall_at_5": 0.286, + "full_recall_at_5": 0.1081, + "mean_recall_at_10": 0.3288, + "full_recall_at_10": 0.1351, + "mean_recall_at_20": 0.3739, + "full_recall_at_20": 0.1892, + "mean_recall_at_1000_tokens": 0.3018, + "mean_recall_at_4000_tokens": 0.3739, + "mean_recall_at_16000_tokens": 0.3739, + "mean_precision_at_10": 0.0806, + "zero_hit_rate": 0.4595, + "median_tokens_delivered": 1697, + "median_latency_ms": 1279.145, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 2580, + "median_latency_ms": 1175.0285, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0.4524, + "full_recall_at_5": 0.1429, + "mean_recall_at_10": 0.4524, + "full_recall_at_10": 0.1429, + "mean_recall_at_20": 0.4524, + "full_recall_at_20": 0.1429, + "mean_recall_at_1000_tokens": 0.4524, + "mean_recall_at_4000_tokens": 0.4524, + "mean_recall_at_16000_tokens": 0.4524, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 1691, + "median_latency_ms": 1224.234, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.2318, + "full_recall_at_5": 0.0938, + "mean_recall_at_10": 0.2813, + "full_recall_at_10": 0.125, + "mean_recall_at_20": 0.3333, + "full_recall_at_20": 0.1875, + "mean_recall_at_1000_tokens": 0.25, + "mean_recall_at_4000_tokens": 0.3333, + "mean_recall_at_16000_tokens": 0.3333, + "mean_precision_at_10": 0.0714, + "zero_hit_rate": 0.5313, + "median_tokens_delivered": 2132, + "median_latency_ms": 1275.568, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 39 + }, + "resources": { + "baseline_rss_mb": 20569.2, + "peak_rss_mb": 21555.4, + "delta_rss_mb": 986.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 48340992 + } + }, + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.3355, + "full_recall_at_5": 0.2051, + "mean_recall_at_10": 0.406, + "full_recall_at_10": 0.2308, + "mean_recall_at_20": 0.4786, + "full_recall_at_20": 0.3077, + "mean_recall_at_1000_tokens": 0.4786, + "mean_recall_at_4000_tokens": 0.4786, + "mean_recall_at_16000_tokens": 0.4786, + "mean_precision_at_10": 0.0844, + "zero_hit_rate": 0.359, + "median_tokens_delivered": 218, + "median_latency_ms": 198.014, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 37, + "mean_recall_at_5": 0.3266, + "full_recall_at_5": 0.1892, + "mean_recall_at_10": 0.4009, + "full_recall_at_10": 0.2162, + "mean_recall_at_20": 0.4775, + "full_recall_at_20": 0.2973, + "mean_recall_at_1000_tokens": 0.4775, + "mean_recall_at_4000_tokens": 0.4775, + "mean_recall_at_16000_tokens": 0.4775, + "mean_precision_at_10": 0.0799, + "zero_hit_rate": 0.3514, + "median_tokens_delivered": 275, + "median_latency_ms": 198.014, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.5, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0.5, + "mean_recall_at_4000_tokens": 0.5, + "mean_recall_at_16000_tokens": 0.5, + "mean_precision_at_10": 0.1667, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 19, + "median_latency_ms": 194.907, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0.4762, + "full_recall_at_5": 0.2857, + "mean_recall_at_10": 0.5952, + "full_recall_at_10": 0.2857, + "mean_recall_at_20": 0.6429, + "full_recall_at_20": 0.4286, + "mean_recall_at_1000_tokens": 0.6429, + "mean_recall_at_4000_tokens": 0.6429, + "mean_recall_at_16000_tokens": 0.6429, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.1429, + "median_tokens_delivered": 275, + "median_latency_ms": 195.879, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.3047, + "full_recall_at_5": 0.1875, + "mean_recall_at_10": 0.3646, + "full_recall_at_10": 0.2188, + "mean_recall_at_20": 0.4427, + "full_recall_at_20": 0.2813, + "mean_recall_at_1000_tokens": 0.4427, + "mean_recall_at_4000_tokens": 0.4427, + "mean_recall_at_16000_tokens": 0.4427, + "mean_precision_at_10": 0.081, + "zero_hit_rate": 0.4063, + "median_tokens_delivered": 190, + "median_latency_ms": 198.834, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 39 + }, + "resources": { + "baseline_rss_mb": 20107.9, + "peak_rss_mb": 20289.5, + "delta_rss_mb": 181.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 339345408 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17705, + "latency_ms": 18.617, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [], + "missed": [ + "src/flask/ctx.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22208, + "latency_ms": 12.57, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30144, + "latency_ms": 12.461, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30503, + "latency_ms": 12.65, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 11009, + "latency_ms": 12.291, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27983, + "latency_ms": 12.635, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "tests/test_cli.py" + ], + "missed": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 26044, + "latency_ms": 12.747, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40575, + "latency_ms": 12.557, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_appctx.py" + ], + "missed": [ + ".flake8", + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 37624, + "latency_ms": 11.854, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py" + ], + "missed": [ + "tests/test_instance_config.py", + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 62976, + "latency_ms": 11.951, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36940, + "latency_ms": 11.917, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_reqctx.py" + ], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 18661, + "latency_ms": 11.716, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 50380, + "latency_ms": 11.23, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30706, + "latency_ms": 11.727, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 45986, + "latency_ms": 14.813, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 32720, + "latency_ms": 14.304, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [ + "src/flask/config.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 28009, + "latency_ms": 11.838, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36384, + "latency_ms": 11.719, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [], + "missed": [ + "src/flask/typing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30334, + "latency_ms": 12.876, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28572, + "latency_ms": 11.8, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43383, + "latency_ms": 12.012, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19148, + "latency_ms": 11.96, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [], + "missed": [ + "pyproject.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33010, + "latency_ms": 12.48, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25252, + "latency_ms": 11.848, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [ + ".pre-commit-config.yaml" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 25471, + "latency_ms": 11.357, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [], + "missed": [ + "src/flask/wrappers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45386, + "latency_ms": 11.508, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23746, + "latency_ms": 11.242, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [ + "CHANGES.rst", + "src/flask/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 36859, + "latency_ms": 11.199, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 36829, + "latency_ms": 11.188, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51711, + "latency_ms": 10.904, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "tests/test_blueprints.py" + ], + "missed": [ + "src/flask/blueprints.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 26267, + "latency_ms": 10.831, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [], + "missed": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23985, + "latency_ms": 11.669, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [], + "missed": [ + "examples/javascript/tests/test_js_example.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40972, + "latency_ms": 11.786, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41679, + "latency_ms": 11.169, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35855, + "latency_ms": 12.384, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "tests/test_views.py" + ], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 2, + "tokens_delivered": 27943, + "latency_ms": 11.121, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19505, + "latency_ms": 11.296, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42482, + "latency_ms": 14.423, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25042, + "latency_ms": 13.251, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 98958, + "latency_ms": 26.947, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 98182, + "latency_ms": 24.828, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 97356, + "latency_ms": 23.359, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 100544, + "latency_ms": 21.876, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [ + "src/flask/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 97421, + "latency_ms": 24.031, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [ + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 70486, + "latency_ms": 23.732, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "src/flask/cli.py", + "tests/test_cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 70069, + "latency_ms": 23.207, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 76284, + "latency_ms": 23.39, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + ".flake8", + "tests/test_appctx.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 101135, + "latency_ms": 22.182, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py" + ], + "missed": [ + "tests/test_instance_config.py", + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 98043, + "latency_ms": 26.918, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70565, + "latency_ms": 24.426, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "tests/test_reqctx.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 100513, + "latency_ms": 26.332, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 72596, + "latency_ms": 24.069, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [ + "src/flask/testing.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 73072, + "latency_ms": 24.291, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 96806, + "latency_ms": 24.605, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 81351, + "latency_ms": 27.166, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [], + "missed": [ + "src/flask/config.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 99251, + "latency_ms": 23.296, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70565, + "latency_ms": 22.547, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [ + "src/flask/typing.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 75828, + "latency_ms": 22.863, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 96787, + "latency_ms": 23.578, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 70795, + "latency_ms": 28.497, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 97421, + "latency_ms": 24.682, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [ + "pyproject.toml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 80328, + "latency_ms": 25.228, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 70553, + "latency_ms": 24.852, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [ + ".pre-commit-config.yaml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 80215, + "latency_ms": 25.935, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [], + "missed": [ + "src/flask/wrappers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70730, + "latency_ms": 24.184, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 99308, + "latency_ms": 24.387, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 97261, + "latency_ms": 24.586, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 95560, + "latency_ms": 25.395, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [ + "src/flask/sessions.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 70731, + "latency_ms": 25.296, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 100533, + "latency_ms": 26.065, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 81260, + "latency_ms": 29.341, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [], + "missed": [ + "examples/javascript/tests/test_js_example.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70553, + "latency_ms": 26.359, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 71520, + "latency_ms": 26.304, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70565, + "latency_ms": 24.474, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_cli.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py", + "tests/test_views.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 80314, + "latency_ms": 24.09, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [ + "src/flask/sessions.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 70461, + "latency_ms": 24.365, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70069, + "latency_ms": 24.495, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [ + "src/flask/json/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 98082, + "latency_ms": 23.552, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4117, + "latency_ms": 611.502, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3762, + "latency_ms": 606.461, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3596, + "latency_ms": 605.776, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4360, + "latency_ms": 617.76, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [ + "src/flask/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 17, + "tokens_delivered": 3687, + "latency_ms": 599.873, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5036, + "latency_ms": 620.752, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_cli.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 3862, + "latency_ms": 649.274, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3664, + "latency_ms": 587.641, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + ".flake8", + "tests/test_appctx.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 8, + "tokens_delivered": 4174, + "latency_ms": 605.082, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py" + ], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 4273, + "latency_ms": 599.348, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4840, + "latency_ms": 613.2, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "tests/test_reqctx.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 3088, + "latency_ms": 610.391, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 3967, + "latency_ms": 717.382, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [ + "src/flask/testing.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3369, + "latency_ms": 743.177, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3724, + "latency_ms": 789.39, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3895, + "latency_ms": 847.284, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [ + "src/flask/config.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 16, + "tokens_delivered": 4757, + "latency_ms": 865.76, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7934, + "latency_ms": 823.585, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [ + "src/flask/typing.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3801, + "latency_ms": 810.485, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 15, + "tokens_delivered": 4246, + "latency_ms": 778.282, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 6024, + "latency_ms": 841.011, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 5053, + "latency_ms": 796.464, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [], + "missed": [ + "pyproject.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3676, + "latency_ms": 839.499, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 6024, + "latency_ms": 870.748, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [], + "missed": [ + ".pre-commit-config.yaml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3383, + "latency_ms": 876.838, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [ + "src/flask/wrappers.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 4053, + "latency_ms": 843.66, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4185, + "latency_ms": 888.93, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3899, + "latency_ms": 844.869, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 15, + "tokens_delivered": 4246, + "latency_ms": 881.49, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [ + "src/flask/sessions.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 5036, + "latency_ms": 789.451, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4723, + "latency_ms": 887.94, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 4088, + "latency_ms": 841.286, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [], + "missed": [ + "examples/javascript/tests/test_js_example.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4738, + "latency_ms": 868.349, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5036, + "latency_ms": 937.613, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4348, + "latency_ms": 917.999, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "src/flask/sansio/app.py", + "tests/test_basic.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 4622, + "latency_ms": 859.89, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [ + "src/flask/sessions.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 6024, + "latency_ms": 851.418, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [ + "src/flask/sansio/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3742, + "latency_ms": 840.717, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [ + "src/flask/json/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 3284, + "latency_ms": 887.275, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 449, + "latency_ms": 673.838, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 473, + "latency_ms": 460.924, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 451, + "latency_ms": 513.53, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 409, + "latency_ms": 474.087, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 542, + "latency_ms": 426.755, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 311, + "latency_ms": 484.482, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 379, + "latency_ms": 518.691, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 474, + "latency_ms": 447.604, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + ".flake8", + "tests/test_appctx.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 437, + "latency_ms": 494.637, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "tests/test_instance_config.py" + ], + "missed": [ + "src/flask/scaffold.py", + "tox.ini" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 440, + "latency_ms": 447.504, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 362, + "latency_ms": 552.159, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "tests/test_reqctx.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 309, + "latency_ms": 434.803, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_blueprints.py" + ], + "recall_at_5": 0.25, + "precision_at_5": 1, + "recall_at_10": 0.25, + "precision_at_10": 1, + "recall_at_20": 0.25, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 655, + "latency_ms": 444.484, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [ + "src/flask/testing.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 391, + "latency_ms": 468.678, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 507, + "latency_ms": 544.682, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 472, + "latency_ms": 507.299, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [], + "missed": [ + "src/flask/config.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 149, + "latency_ms": 502.096, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 578, + "latency_ms": 446.708, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [], + "missed": [ + "src/flask/typing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 467, + "latency_ms": 564.02, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 197, + "latency_ms": 519.995, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 389, + "latency_ms": 484.833, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 384, + "latency_ms": 498.47, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [ + "pyproject.toml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 404, + "latency_ms": 512.678, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 389, + "latency_ms": 452.81, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [ + ".pre-commit-config.yaml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 248, + "latency_ms": 485.276, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [], + "missed": [ + "src/flask/wrappers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 434, + "latency_ms": 509.971, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 425, + "latency_ms": 522.156, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 473, + "latency_ms": 531.44, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 197, + "latency_ms": 640.327, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 311, + "latency_ms": 507.097, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "tests/test_blueprints.py" + ], + "missed": [ + "src/flask/blueprints.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 453, + "latency_ms": 603.586, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [ + "tests/test_testing.py" + ], + "missed": [ + "src/flask/testing.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 457, + "latency_ms": 498.945, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [], + "missed": [ + "examples/javascript/tests/test_js_example.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 364, + "latency_ms": 829.447, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 311, + "latency_ms": 468.81, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 453, + "latency_ms": 511.396, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "recall_at_5": 0.1667, + "precision_at_5": 1, + "recall_at_10": 0.1667, + "precision_at_10": 1, + "recall_at_20": 0.1667, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 1, + "tokens_delivered": 532, + "latency_ms": 499.126, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 389, + "latency_ms": 539.449, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 462, + "latency_ms": 652.769, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [ + "src/flask/json/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 519, + "latency_ms": 521.228, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1727, + "latency_ms": 9541.551, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 3695, + "latency_ms": 17406.418, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [ + "CHANGES.rst" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 2176, + "latency_ms": 15763.463, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3689, + "latency_ms": 61613.241, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [ + "CHANGES.rst" + ], + "missed": [ + "src/flask/__init__.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 3731, + "latency_ms": 129893.49, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3244, + "latency_ms": 17834.813, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "src/flask/cli.py", + "tests/test_cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 10, + "tokens_delivered": 3359, + "latency_ms": 26941.545, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [ + "CHANGES.rst" + ], + "missed": [ + "src/flask/cli.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 14, + "tokens_delivered": 2342, + "latency_ms": 14487.914, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [], + "missed": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2558, + "latency_ms": 18178.112, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "tests/test_instance_config.py" + ], + "missed": [ + "src/flask/scaffold.py", + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 6, + "tokens_delivered": 2204, + "latency_ms": 7028.397, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 240448.933, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1906.983, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 785.762, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 735.3, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 658.535, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 632.398, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [], + "missed": [ + "src/flask/config.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 692.97, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 520.781, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [], + "missed": [ + "src/flask/typing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 744.196, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 787.466, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 677.715, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 779.784, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [], + "missed": [ + "pyproject.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 902.766, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 825.4, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [], + "missed": [ + ".pre-commit-config.yaml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 645.197, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [], + "missed": [ + "src/flask/wrappers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 606.273, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 761.356, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 665.64, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 767.387, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 879.731, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [], + "missed": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 613.32, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [], + "missed": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 829.572, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [], + "missed": [ + "examples/javascript/tests/test_js_example.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 683.676, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 707.471, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 788.732, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 769.558, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 702.001, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 659.907, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 852.278, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1696, + "latency_ms": 2033.654, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1680, + "latency_ms": 1766.619, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7470, + "latency_ms": 1413.456, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1673, + "latency_ms": 1541.492, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [ + "src/flask/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1652, + "latency_ms": 1515.517, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [ + "src/flask/json/provider.py" + ], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2550, + "latency_ms": 1527.868, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_cli.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 287, + "latency_ms": 1509.64, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 4860, + "latency_ms": 1292.825, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [], + "missed": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3947, + "latency_ms": 1217.564, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py" + ], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 1712, + "latency_ms": 1183.672, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1679, + "latency_ms": 1270.881, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1678, + "latency_ms": 1279.145, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1692, + "latency_ms": 1436.938, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [ + "src/flask/testing.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1714, + "latency_ms": 1366.609, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1682, + "latency_ms": 1469.908, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1678, + "latency_ms": 1816.983, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [], + "missed": [ + "src/flask/config.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2610, + "latency_ms": 1176.79, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 800, + "latency_ms": 1246.477, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [], + "missed": [ + "src/flask/typing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1691, + "latency_ms": 1224.234, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1271.991, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1236.569, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1236.91, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [], + "missed": [ + "pyproject.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2610, + "latency_ms": 1303.859, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1258.427, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [], + "missed": [ + ".pre-commit-config.yaml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1173.267, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [ + "src/flask/wrappers.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 1689, + "latency_ms": 1296.197, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1693, + "latency_ms": 1334.834, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 1714, + "latency_ms": 1293.475, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1155.1, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1290.934, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/blueprints.py" + ], + "missed": [ + "tests/test_blueprints.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1697, + "latency_ms": 1343.218, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1677, + "latency_ms": 1214.78, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [ + "examples/javascript/tests/test_js_example.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1684, + "latency_ms": 1199.718, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1226.057, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1207.96, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_views.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_cli.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 1704, + "latency_ms": 1246.929, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1198.82, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [ + "src/flask/sansio/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1671, + "latency_ms": 1190.866, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [ + "src/flask/json/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1668, + "latency_ms": 1139.999, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [ + "CHANGES.rst" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 0, + "latency_ms": 208.48, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 331, + "latency_ms": 199.82, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 121, + "latency_ms": 201.565, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 332, + "latency_ms": 200.244, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [ + "src/flask/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 218, + "latency_ms": 202.284, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31, + "latency_ms": 193.544, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_cli.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 463, + "latency_ms": 190.913, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 538, + "latency_ms": 200.222, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [ + ".flake8", + "tests/test_basic.py" + ], + "missed": [ + "tests/test_appctx.py" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 82, + "latency_ms": 212.906, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 439, + "latency_ms": 191.148, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [ + "tox.ini" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 306, + "latency_ms": 202.313, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "tests/test_reqctx.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 16, + "tokens_delivered": 563, + "latency_ms": 187.619, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "missed": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 131, + "latency_ms": 198.014, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [ + "src/flask/testing.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 355, + "latency_ms": 194.849, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 344, + "latency_ms": 201.077, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 415, + "latency_ms": 192.321, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [], + "missed": [ + "src/flask/config.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7, + "latency_ms": 188.457, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 403, + "latency_ms": 202.626, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [], + "missed": [ + "src/flask/typing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 335, + "latency_ms": 194.195, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20, + "latency_ms": 192.57, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31, + "latency_ms": 207.743, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29, + "latency_ms": 199.13, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [ + "pyproject.toml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 484, + "latency_ms": 198.538, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31, + "latency_ms": 191.702, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [ + ".pre-commit-config.yaml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 31, + "latency_ms": 201.357, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [ + "src/flask/wrappers.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 398, + "latency_ms": 196.724, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 413, + "latency_ms": 197.262, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2222, + "recall_at_20": 0.6667, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 196, + "latency_ms": 211.167, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20, + "latency_ms": 206.069, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31, + "latency_ms": 196.576, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "tests/test_blueprints.py" + ], + "missed": [ + "src/flask/blueprints.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 162, + "latency_ms": 192.18, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 313, + "latency_ms": 198.001, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [ + "examples/javascript/tests/test_js_example.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 275, + "latency_ms": 195.879, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31, + "latency_ms": 199.728, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10, + "latency_ms": 189.4, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_views.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py", + "tests/test_cli.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.5, + "precision_at_20": 0.2727, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 184, + "latency_ms": 194.798, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31, + "latency_ms": 218.135, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [ + "src/flask/sansio/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 364, + "latency_ms": 201.735, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [ + "src/flask/json/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 398, + "latency_ms": 195.977, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/C-gin.json b/benchmarks/context-retrieval/results/C-gin.json new file mode 100644 index 00000000..528b84b0 --- /dev/null +++ b/benchmarks/context-retrieval/results/C-gin.json @@ -0,0 +1,10990 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "tier": "small", + "tier_code_files": 99, + "corpus_counts": { + "cases": 76, + "path_leak": 23, + "baseline_blind": 53, + "multi_file": 54 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.7511, + "control": 0.0263, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "data race with trustedCIDRs (#2674) (#2675)", + "revision": "d496f64540b6707602de50ab57aeea8ff4080b74", + "returned": [ + "tree_test.go", + "binding/form_mapping_test.go", + "binding/form.go", + "routes_test.go", + "tree.go" + ] + }, + { + "provider_id": "random-files", + "query": "Fix #3500 Add escape logic for header (#3503)", + "revision": "fc1c43298de675e5252d0b44f97dc5e204bd4e1e", + "returned": [ + "mode_test.go", + "routes_test.go", + "internal/json/json.go", + "testdata/protoexample/test.pb.go", + "render/render.go" + ] + }, + { + "provider_id": "random-files", + "query": "check obj type in protobufBinding (#2851)", + "revision": "deb83b6365cf1b95f50929492f6058c94a6ff274", + "returned": [ + ".github/workflows/gin.yml", + "tree_test.go", + "render/xml.go", + "binding/protobuf.go", + "test_helpers.go" + ] + }, + { + "provider_id": "churn-ranked", + "query": "data race with trustedCIDRs (#2674) (#2675)", + "revision": "d496f64540b6707602de50ab57aeea8ff4080b74", + "returned": [ + "context.go", + "context_test.go", + "gin.go", + "binding/binding_test.go", + ".travis.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Do not panic when handling method not allowed on empty tree (#4003)", + "revision": "9c081de9cdd1948f521d47d170d18cbc2981c33a", + "returned": [ + "context.go", + "context_test.go", + "gin.go", + ".github/workflows/gin.yml", + "tree.go" + ] + }, + { + "provider_id": "churn-ranked", + "query": "check obj type in protobufBinding (#2851)", + "revision": "deb83b6365cf1b95f50929492f6058c94a6ff274", + "returned": [ + "context.go", + "context_test.go", + "gin.go", + "binding/binding_test.go", + "binding/form_mapping.go" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "data race with trustedCIDRs (#2674) (#2675)", + "revision": "d496f64540b6707602de50ab57aeea8ff4080b74", + "returned": [ + "gin.go", + "context.go", + "gin_integration_test.go", + "gin_test.go", + "githubapi_test.go" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "query binding bug (#3236)", + "revision": "78f4687875d72d10392f8a77008cbefdec4c0aa0", + "returned": [ + ".goreleaser.yaml", + "binding/query.go", + "context.go", + "binding/binding_test.go", + "context_test.go" + ] + }, + { + "provider_id": "graphify", + "query": "data race with trustedCIDRs (#2674) (#2675)", + "revision": "d496f64540b6707602de50ab57aeea8ff4080b74", + "returned": [ + "render/data.go", + "githubapi_test.go", + "gin.go", + "routes_test.go", + "context.go" + ] + }, + { + "provider_id": "graphify", + "query": "sonic on arm64 (#4234)", + "revision": "67c9d4ee110e9adfe33063ef847dba56717c148a", + "returned": [ + "internal/json/sonic.go" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.068, + "full_recall_at_5": 0.0132, + "mean_recall_at_10": 0.0921, + "full_recall_at_10": 0.0263, + "mean_recall_at_20": 0.1776, + "full_recall_at_20": 0.0526, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0263, + "mean_recall_at_16000_tokens": 0.1009, + "mean_precision_at_10": 0.0197, + "zero_hit_rate": 0.6842, + "median_tokens_delivered": 24792.5, + "median_latency_ms": 9.0105, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 74, + "mean_recall_at_5": 0.0563, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0676, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1554, + "full_recall_at_20": 0.027, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.027, + "mean_recall_at_16000_tokens": 0.0766, + "mean_precision_at_10": 0.0162, + "zero_hit_rate": 0.7027, + "median_tokens_delivered": 24792.5, + "median_latency_ms": 9.049, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.15, + "zero_hit_rate": 0, + "median_tokens_delivered": 21401, + "median_latency_ms": 8.5965, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.0435, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0652, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1522, + "full_recall_at_20": 0.0435, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0217, + "mean_recall_at_16000_tokens": 0.0652, + "mean_precision_at_10": 0.013, + "zero_hit_rate": 0.7391, + "median_tokens_delivered": 24180, + "median_latency_ms": 9.056, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.0786, + "full_recall_at_5": 0.0189, + "mean_recall_at_10": 0.1038, + "full_recall_at_10": 0.0377, + "mean_recall_at_20": 0.1887, + "full_recall_at_20": 0.0566, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0283, + "mean_recall_at_16000_tokens": 0.1164, + "mean_precision_at_10": 0.0226, + "zero_hit_rate": 0.6604, + "median_tokens_delivered": 25977, + "median_latency_ms": 8.963, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 76 + }, + "resources": { + "baseline_rss_mb": 21078, + "peak_rss_mb": 21157.5, + "delta_rss_mb": 79.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.3783, + "full_recall_at_5": 0.2632, + "mean_recall_at_10": 0.5077, + "full_recall_at_10": 0.3553, + "mean_recall_at_20": 0.7368, + "full_recall_at_20": 0.6579, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0175, + "mean_recall_at_16000_tokens": 0.2884, + "mean_precision_at_10": 0.0934, + "zero_hit_rate": 0.1842, + "median_tokens_delivered": 88454.5, + "median_latency_ms": 19.0745, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 74, + "mean_recall_at_5": 0.375, + "full_recall_at_5": 0.2568, + "mean_recall_at_10": 0.5079, + "full_recall_at_10": 0.3514, + "mean_recall_at_20": 0.7297, + "full_recall_at_20": 0.6486, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.018, + "mean_recall_at_16000_tokens": 0.2962, + "mean_precision_at_10": 0.0946, + "zero_hit_rate": 0.1892, + "median_tokens_delivered": 88454.5, + "median_latency_ms": 19.0745, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.05, + "zero_hit_rate": 0, + "median_tokens_delivered": 87302.5, + "median_latency_ms": 18.944, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.4203, + "full_recall_at_5": 0.3478, + "mean_recall_at_10": 0.5507, + "full_recall_at_10": 0.3913, + "mean_recall_at_20": 0.7174, + "full_recall_at_20": 0.6087, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0435, + "mean_recall_at_16000_tokens": 0.3768, + "mean_precision_at_10": 0.0913, + "zero_hit_rate": 0.1739, + "median_tokens_delivered": 90059, + "median_latency_ms": 19.122, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3601, + "full_recall_at_5": 0.2264, + "mean_recall_at_10": 0.489, + "full_recall_at_10": 0.3396, + "mean_recall_at_20": 0.7453, + "full_recall_at_20": 0.6792, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0063, + "mean_recall_at_16000_tokens": 0.25, + "mean_precision_at_10": 0.0943, + "zero_hit_rate": 0.1887, + "median_tokens_delivered": 88304, + "median_latency_ms": 18.981, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 76 + }, + "resources": { + "baseline_rss_mb": 21173.9, + "peak_rss_mb": 21252.5, + "delta_rss_mb": 78.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.5779, + "full_recall_at_5": 0.4474, + "mean_recall_at_10": 0.6579, + "full_recall_at_10": 0.5395, + "mean_recall_at_20": 0.7303, + "full_recall_at_20": 0.6579, + "mean_recall_at_1000_tokens": 0.7237, + "mean_recall_at_4000_tokens": 0.7303, + "mean_recall_at_16000_tokens": 0.7303, + "mean_precision_at_10": 0.1248, + "zero_hit_rate": 0.1974, + "median_tokens_delivered": 390, + "median_latency_ms": 159.0335, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 74, + "mean_recall_at_5": 0.5664, + "full_recall_at_5": 0.4324, + "mean_recall_at_10": 0.6486, + "full_recall_at_10": 0.527, + "mean_recall_at_20": 0.723, + "full_recall_at_20": 0.6486, + "mean_recall_at_1000_tokens": 0.7162, + "mean_recall_at_4000_tokens": 0.723, + "mean_recall_at_16000_tokens": 0.723, + "mean_precision_at_10": 0.1241, + "zero_hit_rate": 0.2027, + "median_tokens_delivered": 382.5, + "median_latency_ms": 159.0335, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 1, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.15, + "zero_hit_rate": 0, + "median_tokens_delivered": 582.5, + "median_latency_ms": 167.803, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.7645, + "full_recall_at_5": 0.6087, + "mean_recall_at_10": 0.808, + "full_recall_at_10": 0.6522, + "mean_recall_at_20": 0.8515, + "full_recall_at_20": 0.7391, + "mean_recall_at_1000_tokens": 0.8515, + "mean_recall_at_4000_tokens": 0.8515, + "mean_recall_at_16000_tokens": 0.8515, + "mean_precision_at_10": 0.1478, + "zero_hit_rate": 0.0435, + "median_tokens_delivered": 399, + "median_latency_ms": 159.872, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.4969, + "full_recall_at_5": 0.3774, + "mean_recall_at_10": 0.5928, + "full_recall_at_10": 0.4906, + "mean_recall_at_20": 0.6777, + "full_recall_at_20": 0.6226, + "mean_recall_at_1000_tokens": 0.6682, + "mean_recall_at_4000_tokens": 0.6777, + "mean_recall_at_16000_tokens": 0.6777, + "mean_precision_at_10": 0.1148, + "zero_hit_rate": 0.2642, + "median_tokens_delivered": 376, + "median_latency_ms": 158.188, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 73, + "indexed-but-returned-nothing": 3 + }, + "resources": { + "baseline_rss_mb": 21271.8, + "peak_rss_mb": 21577.3, + "delta_rss_mb": 305.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 703463424 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.4583, + "full_recall_at_5": 0.3026, + "mean_recall_at_10": 0.5417, + "full_recall_at_10": 0.4211, + "mean_recall_at_20": 0.5789, + "full_recall_at_20": 0.4342, + "mean_recall_at_1000_tokens": 0.4342, + "mean_recall_at_4000_tokens": 0.568, + "mean_recall_at_16000_tokens": 0.5789, + "mean_precision_at_10": 0.1672, + "zero_hit_rate": 0.2632, + "median_tokens_delivered": 1671, + "median_latency_ms": 774.844, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 74, + "mean_recall_at_5": 0.4707, + "full_recall_at_5": 0.3108, + "mean_recall_at_10": 0.5428, + "full_recall_at_10": 0.4189, + "mean_recall_at_20": 0.5743, + "full_recall_at_20": 0.4324, + "mean_recall_at_1000_tokens": 0.4459, + "mean_recall_at_4000_tokens": 0.5631, + "mean_recall_at_16000_tokens": 0.5743, + "mean_precision_at_10": 0.1698, + "zero_hit_rate": 0.2703, + "median_tokens_delivered": 1671, + "median_latency_ms": 774.844, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.75, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.75, + "mean_recall_at_16000_tokens": 0.75, + "mean_precision_at_10": 0.0715, + "zero_hit_rate": 0, + "median_tokens_delivered": 2731.5, + "median_latency_ms": 778.893, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.6196, + "full_recall_at_5": 0.4348, + "mean_recall_at_10": 0.7283, + "full_recall_at_10": 0.6087, + "mean_recall_at_20": 0.7283, + "full_recall_at_20": 0.6087, + "mean_recall_at_1000_tokens": 0.7065, + "mean_recall_at_4000_tokens": 0.7283, + "mean_recall_at_16000_tokens": 0.7283, + "mean_precision_at_10": 0.1979, + "zero_hit_rate": 0.1304, + "median_tokens_delivered": 1666, + "median_latency_ms": 780.897, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3884, + "full_recall_at_5": 0.2453, + "mean_recall_at_10": 0.4607, + "full_recall_at_10": 0.3396, + "mean_recall_at_20": 0.5142, + "full_recall_at_20": 0.3585, + "mean_recall_at_1000_tokens": 0.316, + "mean_recall_at_4000_tokens": 0.4984, + "mean_recall_at_16000_tokens": 0.5142, + "mean_precision_at_10": 0.1538, + "zero_hit_rate": 0.3208, + "median_tokens_delivered": 1676, + "median_latency_ms": 773.965, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 75, + "indexed-but-returned-nothing": 1 + }, + "resources": { + "baseline_rss_mb": 21070.2, + "peak_rss_mb": 22132.3, + "delta_rss_mb": 1062.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 117551104 + } + }, + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.4616, + "full_recall_at_5": 0.1974, + "mean_recall_at_10": 0.6206, + "full_recall_at_10": 0.3947, + "mean_recall_at_20": 0.7577, + "full_recall_at_20": 0.5921, + "mean_recall_at_1000_tokens": 0.4397, + "mean_recall_at_4000_tokens": 0.7511, + "mean_recall_at_16000_tokens": 0.7577, + "mean_precision_at_10": 0.1118, + "zero_hit_rate": 0.0921, + "median_tokens_delivered": 3848, + "median_latency_ms": 513.442, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 74, + "mean_recall_at_5": 0.4538, + "full_recall_at_5": 0.1892, + "mean_recall_at_10": 0.6171, + "full_recall_at_10": 0.3919, + "mean_recall_at_20": 0.7579, + "full_recall_at_20": 0.5946, + "mean_recall_at_1000_tokens": 0.4313, + "mean_recall_at_4000_tokens": 0.7511, + "mean_recall_at_16000_tokens": 0.7579, + "mean_precision_at_10": 0.1122, + "zero_hit_rate": 0.0946, + "median_tokens_delivered": 3848, + "median_latency_ms": 513.442, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.75, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.75, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.75, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0.75, + "mean_recall_at_4000_tokens": 0.75, + "mean_recall_at_16000_tokens": 0.75, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 3733, + "median_latency_ms": 512.435, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.6594, + "full_recall_at_5": 0.3913, + "mean_recall_at_10": 0.7681, + "full_recall_at_10": 0.5217, + "mean_recall_at_20": 0.8116, + "full_recall_at_20": 0.6087, + "mean_recall_at_1000_tokens": 0.6087, + "mean_recall_at_4000_tokens": 0.8116, + "mean_recall_at_16000_tokens": 0.8116, + "mean_precision_at_10": 0.1304, + "zero_hit_rate": 0, + "median_tokens_delivered": 3845, + "median_latency_ms": 513.355, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3758, + "full_recall_at_5": 0.1132, + "mean_recall_at_10": 0.5566, + "full_recall_at_10": 0.3396, + "mean_recall_at_20": 0.7343, + "full_recall_at_20": 0.5849, + "mean_recall_at_1000_tokens": 0.3664, + "mean_recall_at_4000_tokens": 0.7248, + "mean_recall_at_16000_tokens": 0.7343, + "mean_precision_at_10": 0.1038, + "zero_hit_rate": 0.1321, + "median_tokens_delivered": 3851, + "median_latency_ms": 513.738, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 76 + }, + "resources": { + "baseline_rss_mb": 20690.1, + "peak_rss_mb": 21959.3, + "delta_rss_mb": 1269.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go", + "logger_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 44600, + "latency_ms": 20.898, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12761, + "latency_ms": 11.848, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26285, + "latency_ms": 10.763, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24674, + "latency_ms": 9.415, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21162, + "latency_ms": 10.302, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 27850, + "latency_ms": 9.297, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28556, + "latency_ms": 9.317, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18035, + "latency_ms": 9.293, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19033, + "latency_ms": 9.14, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 37122, + "latency_ms": 8.742, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30316, + "latency_ms": 9.606, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 33394, + "latency_ms": 8.661, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26404, + "latency_ms": 8.569, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26292, + "latency_ms": 9.626, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22473, + "latency_ms": 8.649, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 15956, + "latency_ms": 9.979, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19388, + "latency_ms": 8.97, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17445, + "latency_ms": 9.298, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go" + ], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 27773, + "latency_ms": 8.963, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14497, + "latency_ms": 9.206, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30553, + "latency_ms": 11.107, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 37728, + "latency_ms": 11.044, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 34527, + "latency_ms": 9.707, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19547, + "latency_ms": 8.723, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12483, + "latency_ms": 8.783, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 22479, + "latency_ms": 8.839, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14254, + "latency_ms": 8.943, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 35496, + "latency_ms": 9.328, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [], + "missed": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23550, + "latency_ms": 9.219, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18818, + "latency_ms": 8.903, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 48958, + "latency_ms": 9.042, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53000, + "latency_ms": 8.641, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 28956, + "latency_ms": 8.577, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21545, + "latency_ms": 8.62, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 34999, + "latency_ms": 9.215, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 26477, + "latency_ms": 8.842, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20935, + "latency_ms": 10.039, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20327, + "latency_ms": 8.979, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21566, + "latency_ms": 10.098, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25538, + "latency_ms": 8.595, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24170, + "latency_ms": 8.703, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23086, + "latency_ms": 8.573, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20136, + "latency_ms": 9.453, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8753, + "latency_ms": 8.479, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35582, + "latency_ms": 9.267, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 27255, + "latency_ms": 8.562, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30122, + "latency_ms": 8.537, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [], + "missed": [ + "gin.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19879, + "latency_ms": 10.525, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33426, + "latency_ms": 8.323, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16642, + "latency_ms": 8.482, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27618, + "latency_ms": 11.474, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "errors.go", + "routergroup.go" + ], + "missed": [ + "context.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 39163, + "latency_ms": 10.594, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 16118, + "latency_ms": 9.056, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23279, + "latency_ms": 9.152, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 36634, + "latency_ms": 9.204, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 40307, + "latency_ms": 9.184, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23860, + "latency_ms": 9.327, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37903, + "latency_ms": 9.131, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49541, + "latency_ms": 9.455, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44167, + "latency_ms": 8.788, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13012, + "latency_ms": 9.168, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28129, + "latency_ms": 8.662, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19778, + "latency_ms": 8.312, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 27314, + "latency_ms": 8.554, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22773, + "latency_ms": 8.561, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 28023, + "latency_ms": 9.248, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24288, + "latency_ms": 8.814, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26606, + "latency_ms": 8.407, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21083, + "latency_ms": 8.849, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8705, + "latency_ms": 9.11, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55452, + "latency_ms": 8.476, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 25977, + "latency_ms": 8.638, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/protobuf.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 24180, + "latency_ms": 8.514, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16819, + "latency_ms": 8.743, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 15488, + "latency_ms": 8.639, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24911, + "latency_ms": 8.715, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 69810, + "latency_ms": 26.007, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 84576, + "latency_ms": 20.984, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87509, + "latency_ms": 19.264, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 82399, + "latency_ms": 19.446, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 83992, + "latency_ms": 19.252, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 72743, + "latency_ms": 18.674, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 88246, + "latency_ms": 18.623, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105412, + "latency_ms": 19.027, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 84449, + "latency_ms": 17.823, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 95317, + "latency_ms": 18.176, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 69979, + "latency_ms": 18.062, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 113276, + "latency_ms": 19.426, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 108509, + "latency_ms": 19.425, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 84446, + "latency_ms": 19.173, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 90304, + "latency_ms": 22.346, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 89875, + "latency_ms": 20.265, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70138, + "latency_ms": 18.981, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93950, + "latency_ms": 18.277, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 87314, + "latency_ms": 18.237, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82768, + "latency_ms": 18.952, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 88101, + "latency_ms": 19.152, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 87276, + "latency_ms": 18.849, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 117055, + "latency_ms": 18.361, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 113658, + "latency_ms": 18.639, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 87849, + "latency_ms": 17.972, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 89097, + "latency_ms": 17.977, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93972, + "latency_ms": 17.685, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 88010, + "latency_ms": 18.677, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "test_helpers.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 88138, + "latency_ms": 20.999, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 103341, + "latency_ms": 21.161, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 115951, + "latency_ms": 19.144, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 106775, + "latency_ms": 18.371, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 94928, + "latency_ms": 19.249, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 110221, + "latency_ms": 18.406, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87531, + "latency_ms": 19.974, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 102446, + "latency_ms": 19.222, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 83359, + "latency_ms": 19.122, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 66540, + "latency_ms": 17.953, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 101365, + "latency_ms": 18.339, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 88522, + "latency_ms": 18.343, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 92380, + "latency_ms": 18.893, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 82724, + "latency_ms": 23.322, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 88010, + "latency_ms": 20.841, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 84577, + "latency_ms": 22.542, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 88304, + "latency_ms": 20.223, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 88116, + "latency_ms": 19.319, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 98912, + "latency_ms": 19.591, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 86657, + "latency_ms": 18.285, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 90329, + "latency_ms": 18.411, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88014, + "latency_ms": 19.322, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 112651, + "latency_ms": 18.374, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "recovery_test.go" + ], + "missed": [ + "errors.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 115859, + "latency_ms": 18.634, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88387, + "latency_ms": 18.395, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 82791, + "latency_ms": 18.85, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 111258, + "latency_ms": 18.27, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88075, + "latency_ms": 19.884, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86659, + "latency_ms": 20.127, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 88259, + "latency_ms": 21.276, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 101364, + "latency_ms": 20.057, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 108196, + "latency_ms": 19.737, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 90059, + "latency_ms": 18.746, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [ + "recovery.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 72362, + "latency_ms": 18.728, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 105612, + "latency_ms": 18.844, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 106371, + "latency_ms": 19.407, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 92820, + "latency_ms": 18.703, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 83426, + "latency_ms": 17.992, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 84119, + "latency_ms": 19.322, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 113276, + "latency_ms": 18.866, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 113946, + "latency_ms": 19.401, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93998, + "latency_ms": 19.528, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 98912, + "latency_ms": 20.429, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 98262, + "latency_ms": 20.944, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "binding/protobuf.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 82704, + "latency_ms": 19.4, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 99455, + "latency_ms": 19.479, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 68234, + "latency_ms": 18.481, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 90299, + "latency_ms": 18.477, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [ + "logger_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 265, + "latency_ms": 152.094, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 725, + "latency_ms": 158.188, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 399, + "latency_ms": 153.146, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 580, + "latency_ms": 159.931, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 162.188, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 329, + "latency_ms": 142.8, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 153.102, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 180, + "latency_ms": 179.476, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 153.955, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 271, + "latency_ms": 163.01, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 577, + "latency_ms": 139.553, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 364, + "latency_ms": 189.611, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 80, + "latency_ms": 180.506, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 359, + "latency_ms": 157.042, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 258, + "latency_ms": 159.872, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 433, + "latency_ms": 153.383, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1630, + "latency_ms": 152.178, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 573, + "latency_ms": 161.235, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 140, + "latency_ms": 160.05, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 533, + "latency_ms": 150.269, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 425, + "latency_ms": 154.793, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 484, + "latency_ms": 156.175, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 502, + "latency_ms": 185.987, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 458, + "latency_ms": 192.702, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1604, + "latency_ms": 159.583, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 391, + "latency_ms": 154.157, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 234, + "latency_ms": 160.82, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 194, + "latency_ms": 155.486, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "test_helpers.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 553, + "latency_ms": 156.964, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 196, + "latency_ms": 174.792, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 389, + "latency_ms": 188.503, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 674, + "latency_ms": 188.855, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 18, + "tokens_delivered": 828, + "latency_ms": 158.858, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1249, + "latency_ms": 180.903, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 531, + "latency_ms": 159.209, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 495, + "latency_ms": 166.97, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 971, + "latency_ms": 146.59, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 120, + "latency_ms": 142.263, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 368, + "latency_ms": 174.524, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 557, + "latency_ms": 152.838, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "mode.go", + "mode_test.go" + ], + "missed": [ + "debug.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 410, + "latency_ms": 163.075, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 663, + "latency_ms": 145.125, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 19, + "tokens_delivered": 775, + "latency_ms": 153.443, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 245, + "latency_ms": 149.644, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 344, + "latency_ms": 152.913, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 439, + "latency_ms": 156.738, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 463, + "latency_ms": 168.972, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 257, + "latency_ms": 150.141, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 362, + "latency_ms": 154.956, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 237, + "latency_ms": 149.728, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1671, + "latency_ms": 192.815, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [ + "context.go", + "errors.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 6, + "tokens_delivered": 743, + "latency_ms": 188.806, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 668, + "latency_ms": 147.644, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 333, + "latency_ms": 153.181, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 314, + "latency_ms": 184.973, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 222, + "latency_ms": 156.321, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 177, + "latency_ms": 148.888, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 962, + "latency_ms": 161.95, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go", + "errors_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 179, + "latency_ms": 171.2, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 236, + "latency_ms": 188.561, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 518, + "latency_ms": 155.337, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 176, + "latency_ms": 146.464, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 145, + "latency_ms": 187.559, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 589, + "latency_ms": 186.425, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 163, + "latency_ms": 166.087, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 231, + "latency_ms": 154.469, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 376, + "latency_ms": 148.199, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 365, + "latency_ms": 202.332, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 443, + "latency_ms": 200.921, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 564, + "latency_ms": 172.843, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 314, + "latency_ms": 173.631, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 531, + "latency_ms": 174.203, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 531, + "latency_ms": 154.685, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [ + ".golangci.yml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 114, + "latency_ms": 173.613, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 576, + "latency_ms": 149.181, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 275, + "latency_ms": 168.543, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [ + "logger_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 5, + "tokens_delivered": 6018, + "latency_ms": 836.701, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 1676, + "latency_ms": 760.824, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1664, + "latency_ms": 791.561, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1666, + "latency_ms": 827.867, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 6022, + "latency_ms": 1033.922, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 7862, + "latency_ms": 798.636, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4075, + "latency_ms": 750.137, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1673, + "latency_ms": 842.456, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7, + "latency_ms": 786.981, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 223, + "latency_ms": 794.518, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 668, + "latency_ms": 708.487, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 192, + "latency_ms": 895.379, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82, + "latency_ms": 847.821, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1669, + "latency_ms": 755.745, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1660, + "latency_ms": 793.641, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4145, + "latency_ms": 754.672, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8733, + "latency_ms": 715.438, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 1674, + "latency_ms": 800.991, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go", + "tree_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 161, + "latency_ms": 784.521, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1678, + "latency_ms": 726.639, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4283, + "latency_ms": 761.33, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 496, + "latency_ms": 766.478, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1663, + "latency_ms": 887.15, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 1677, + "latency_ms": 908.861, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2269, + "latency_ms": 768.596, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1665, + "latency_ms": 754.446, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2061, + "latency_ms": 765.345, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.5, + "precision_at_20": 0.3, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1667, + "latency_ms": 753.203, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.1579, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2706, + "latency_ms": 760.88, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 485, + "latency_ms": 824.557, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1664, + "latency_ms": 877.401, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1671, + "latency_ms": 848.847, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1671, + "latency_ms": 781.391, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1675, + "latency_ms": 845.391, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2333, + "latency_ms": 780.897, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8411, + "latency_ms": 801.455, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1666, + "latency_ms": 751.203, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1656, + "latency_ms": 707.91, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 1, + "recall_at_10": 0.25, + "precision_at_10": 1, + "recall_at_20": 0.25, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 130, + "latency_ms": 802.644, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1662, + "latency_ms": 762.477, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1670, + "latency_ms": 788.46, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 146, + "latency_ms": 733.877, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 19, + "tokens_delivered": 1675, + "latency_ms": 775.723, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5798, + "latency_ms": 738.058, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/validate_test.go" + ], + "missed": [ + "binding/default_validator.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2608, + "latency_ms": 763.433, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1668, + "latency_ms": 769.475, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1139, + "latency_ms": 803.48, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1670, + "latency_ms": 742.781, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1040, + "latency_ms": 767.876, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 211, + "latency_ms": 749.974, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1707, + "latency_ms": 899.256, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 4, + "tokens_delivered": 3336, + "latency_ms": 881.099, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1664, + "latency_ms": 756.884, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1664, + "latency_ms": 743.593, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 6602, + "latency_ms": 865.588, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4366, + "latency_ms": 761.088, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1678, + "latency_ms": 734.07, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 1668, + "latency_ms": 750.691, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1666, + "latency_ms": 800.521, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6296, + "latency_ms": 854.34, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1667, + "latency_ms": 762.512, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1668, + "latency_ms": 744.604, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1683, + "latency_ms": 840.695, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 3782, + "latency_ms": 846.854, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 194, + "latency_ms": 763.443, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3970, + "latency_ms": 766.865, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1688, + "latency_ms": 723.981, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1671, + "latency_ms": 882.98, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 5273, + "latency_ms": 890.696, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 6577, + "latency_ms": 773.965, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1669, + "latency_ms": 789.589, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5820, + "latency_ms": 791.94, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/protobuf.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 7343, + "latency_ms": 741.237, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7724, + "latency_ms": 793.416, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug_test.go" + ], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 20, + "tokens_delivered": 1681, + "latency_ms": 710.932, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1666, + "latency_ms": 766.464, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [ + "logger_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3949, + "latency_ms": 693.772, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "debug_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3413, + "latency_ms": 503.024, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4196, + "latency_ms": 513.529, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3356, + "latency_ms": 476.841, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3341, + "latency_ms": 485.357, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3915, + "latency_ms": 473.025, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 17, + "tokens_delivered": 3102, + "latency_ms": 504.504, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3950, + "latency_ms": 541.996, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3594, + "latency_ms": 513.738, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3675, + "latency_ms": 536.921, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4148, + "latency_ms": 482.325, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4239, + "latency_ms": 609.12, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3553, + "latency_ms": 582.101, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3938, + "latency_ms": 721.697, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3845, + "latency_ms": 760.902, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3792, + "latency_ms": 503.978, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [ + "fs.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 19, + "tokens_delivered": 3517, + "latency_ms": 470.774, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 4318, + "latency_ms": 519.685, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3808, + "latency_ms": 508.445, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3942, + "latency_ms": 484.152, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 3939, + "latency_ms": 507.362, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 4008, + "latency_ms": 503.634, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3618, + "latency_ms": 577.85, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3892, + "latency_ms": 579.742, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3630, + "latency_ms": 502.618, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 3801, + "latency_ms": 508.022, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3704, + "latency_ms": 524.899, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "errors.go", + "gin.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "context_test.go", + "response_writer.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 3726, + "latency_ms": 509.433, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2143, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3829, + "latency_ms": 506.163, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 3720, + "latency_ms": 548.793, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go" + ], + "missed": [ + "render/render_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3768, + "latency_ms": 570.013, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4121, + "latency_ms": 551.954, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3904, + "latency_ms": 515.624, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4266, + "latency_ms": 582.427, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4026, + "latency_ms": 513.355, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3749, + "latency_ms": 532.105, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4030, + "latency_ms": 499.06, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3415, + "latency_ms": 461.861, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3930, + "latency_ms": 533.59, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3715, + "latency_ms": 501.146, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go" + ], + "missed": [ + "mode_test.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3663, + "latency_ms": 520.689, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4155, + "latency_ms": 489.947, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3024, + "latency_ms": 505.568, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 3276, + "latency_ms": 496.181, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go" + ], + "missed": [ + "binding/validate_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4094, + "latency_ms": 506.266, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 4139, + "latency_ms": 502.78, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4014, + "latency_ms": 528.332, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4052, + "latency_ms": 490.556, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4032, + "latency_ms": 506.916, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3851, + "latency_ms": 506.615, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4035, + "latency_ms": 573.848, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "errors.go", + "routergroup.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 4, + "tokens_delivered": 3942, + "latency_ms": 585.564, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3626, + "latency_ms": 511.551, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3949, + "latency_ms": 488.811, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3891, + "latency_ms": 566.618, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3804, + "latency_ms": 518.444, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3933, + "latency_ms": 495.173, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3626, + "latency_ms": 500.861, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go" + ], + "missed": [ + "errors_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4241, + "latency_ms": 564.325, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3656, + "latency_ms": 564.997, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3696, + "latency_ms": 509.775, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [ + "recovery.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 16, + "tokens_delivered": 3949, + "latency_ms": 472.698, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3892, + "latency_ms": 547.823, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4230, + "latency_ms": 558.085, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3704, + "latency_ms": 514.252, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 16, + "tokens_delivered": 3800, + "latency_ms": 497.679, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3741, + "latency_ms": 495.899, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4189, + "latency_ms": 584.862, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3931, + "latency_ms": 571.82, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3774, + "latency_ms": 515.347, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3918, + "latency_ms": 535.617, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3780, + "latency_ms": 521.182, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3870, + "latency_ms": 483.514, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3844, + "latency_ms": 528.556, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "debug_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3236, + "latency_ms": 466.785, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 3146, + "latency_ms": 517.464, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/C-got.json b/benchmarks/context-retrieval/results/C-got.json new file mode 100644 index 00000000..e7b3d2f9 --- /dev/null +++ b/benchmarks/context-retrieval/results/C-got.json @@ -0,0 +1,21685 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.8357, + "control": 0.0296, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "source/as-promise/core.ts", + "source/as-promise/calculate-retry-delay.ts", + "source/core/utils/timed-out.ts", + "test/cookies.ts", + "test/types/create-test-server/index.d.ts" + ] + }, + { + "provider_id": "random-files", + "query": "Fix hook type definitions to reflect normalized runtime state", + "revision": "60a441939af36143d1ead87b79ba706fe0a5b812", + "returned": [ + "source/core/utils/is-form-data.ts", + "documentation/examples/runkit-example.js", + "test/helpers/with-server.ts", + "test/helpers/create-https-test-server.ts", + "source/core/parse-link-header.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "package.json", + "source/core/index.ts", + "source/create.ts", + "test/timeout.ts", + "test/arguments.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Handle abort signals added by handlers", + "revision": "f6a058a7d1fdd0b65bb75db9faf94490fb7a66ec", + "returned": [ + "source/core/index.ts", + "package.json", + "source/core/options.ts", + "test/hooks.ts", + "test/stream.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix TypeScript type inference for `got.extend()` with `responseType` option", + "revision": "a149dfbf72fedc3822cc9f8157582f6b92d43774", + "returned": [ + "source/core/index.ts", + "package.json", + "source/core/options.ts", + "test/stream.ts", + "test/hooks.ts" + ] + }, + { + "provider_id": "semble", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "source/core/index.ts", + "source/core/utils/dns-ip-version.ts", + "source/index.ts", + "source/create.ts", + "benchmark/index.ts" + ] + }, + { + "provider_id": "semble", + "query": "Fix `Invalid URL` checks on Node.js 16", + "revision": "a06e7585ab4503e1dd7178d83dd4443ff6dda6c1", + "returned": [ + "source/core/options.ts", + "benchmark/index.ts", + "source/core/index.ts", + "source/core/errors.ts", + "source/create.ts" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix assertions when cloning raw options", + "revision": "0fb6ec60d299fd9b48966608a4c3f201746d821c", + "returned": [ + "source/core/options.ts", + "source/core/index.ts", + "test/helpers/with-server.ts" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix unhandled `The server aborted pending request` rejection", + "revision": "04f3ea4b7995e1609555dd13f00641b8f470de4c", + "returned": [ + "test/cancel.ts", + "source/core/index.ts", + "test/helpers/with-server.ts" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "readme.md", + "package.json", + "source/core/index.ts", + "source/index.ts", + "test/types/slow-stream/index.d.ts" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "Fix path segments containing colons being misidentified as absolute URLs", + "revision": "ff46eaecba4303dc71034552c70c31718b0e9807", + "returned": [ + "package.json", + "readme.md", + "documentation/typescript.md", + "source/core/options.ts", + "documentation/migration-guides/request.md" + ] + }, + { + "provider_id": "graphify", + "query": "Fix encoding with json `responseType` (#1996)", + "revision": "dbbd31777b3299ceb0db99798453e01ab8e597b1", + "returned": [ + "source/core/options.ts", + "documentation/2-options.md", + "source/core/response.ts", + "source/core/index.ts", + "source/index.ts" + ] + }, + { + "provider_id": "graphify", + "query": "Fix unhandled `The server aborted pending request` rejection", + "revision": "04f3ea4b7995e1609555dd13f00641b8f470de4c", + "returned": [ + "readme.md", + "source/as-promise/create-rejection.ts", + "benchmark/server.ts", + "source/create.ts", + "source/as-promise/types.ts" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "test/timeout.ts", + "test/cache.ts", + "source/core/index.ts", + "test/arguments.ts", + "source/core/utils/dns-ip-version.ts" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.0968, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.1927, + "full_recall_at_10": 0.0741, + "mean_recall_at_20": 0.359, + "full_recall_at_20": 0.1481, + "mean_recall_at_1000_tokens": 0.0031, + "mean_recall_at_4000_tokens": 0.0296, + "mean_recall_at_16000_tokens": 0.1954, + "mean_precision_at_10": 0.0361, + "zero_hit_rate": 0.4167, + "median_tokens_delivered": 29954, + "median_latency_ms": 11.0875, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 100, + "mean_recall_at_5": 0.0645, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1282, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3077, + "full_recall_at_20": 0.08, + "mean_recall_at_1000_tokens": 0.0033, + "mean_recall_at_4000_tokens": 0.012, + "mean_recall_at_16000_tokens": 0.131, + "mean_precision_at_10": 0.031, + "zero_hit_rate": 0.45, + "median_tokens_delivered": 29656.5, + "median_latency_ms": 11.0945, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 8, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.25, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 34611, + "median_latency_ms": 10.617, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.1302, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.242, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.3753, + "full_recall_at_20": 0.1667, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.016, + "mean_recall_at_16000_tokens": 0.2519, + "mean_precision_at_10": 0.0426, + "zero_hit_rate": 0.4074, + "median_tokens_delivered": 29934, + "median_latency_ms": 11.034, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.0633, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.1435, + "full_recall_at_10": 0.037, + "mean_recall_at_20": 0.3426, + "full_recall_at_20": 0.1296, + "mean_recall_at_1000_tokens": 0.0062, + "mean_recall_at_4000_tokens": 0.0432, + "mean_recall_at_16000_tokens": 0.1389, + "mean_precision_at_10": 0.0296, + "zero_hit_rate": 0.4259, + "median_tokens_delivered": 29954, + "median_latency_ms": 11.1015, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 15739.8, + "peak_rss_mb": 15816.6, + "delta_rss_mb": 76.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.4293, + "full_recall_at_5": 0.1852, + "mean_recall_at_10": 0.5823, + "full_recall_at_10": 0.3426, + "mean_recall_at_20": 0.7793, + "full_recall_at_20": 0.6019, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0278, + "mean_recall_at_16000_tokens": 0.3735, + "mean_precision_at_10": 0.1157, + "zero_hit_rate": 0.0741, + "median_tokens_delivered": 76435, + "median_latency_ms": 24.974, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 100, + "mean_recall_at_5": 0.4437, + "full_recall_at_5": 0.18, + "mean_recall_at_10": 0.5888, + "full_recall_at_10": 0.33, + "mean_recall_at_20": 0.7817, + "full_recall_at_20": 0.59, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.01, + "mean_recall_at_16000_tokens": 0.3833, + "mean_precision_at_10": 0.121, + "zero_hit_rate": 0.06, + "median_tokens_delivered": 76435, + "median_latency_ms": 24.996, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 8, + "mean_recall_at_5": 0.25, + "full_recall_at_5": 0.25, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.75, + "full_recall_at_20": 0.75, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.25, + "mean_recall_at_16000_tokens": 0.25, + "mean_precision_at_10": 0.05, + "zero_hit_rate": 0.25, + "median_tokens_delivered": 75185.5, + "median_latency_ms": 24.792, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3401, + "full_recall_at_5": 0.0926, + "mean_recall_at_10": 0.4963, + "full_recall_at_10": 0.2037, + "mean_recall_at_20": 0.7161, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0093, + "mean_recall_at_16000_tokens": 0.2778, + "mean_precision_at_10": 0.1056, + "zero_hit_rate": 0.0926, + "median_tokens_delivered": 76984, + "median_latency_ms": 24.5735, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.5185, + "full_recall_at_5": 0.2778, + "mean_recall_at_10": 0.6682, + "full_recall_at_10": 0.4815, + "mean_recall_at_20": 0.8426, + "full_recall_at_20": 0.7037, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0463, + "mean_recall_at_16000_tokens": 0.4691, + "mean_precision_at_10": 0.1259, + "zero_hit_rate": 0.0556, + "median_tokens_delivered": 75083.5, + "median_latency_ms": 25.138, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 15830.3, + "peak_rss_mb": 16390.5, + "delta_rss_mb": 560.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.5755, + "full_recall_at_5": 0.2315, + "mean_recall_at_10": 0.7452, + "full_recall_at_10": 0.5463, + "mean_recall_at_20": 0.8495, + "full_recall_at_20": 0.7407, + "mean_recall_at_1000_tokens": 0.4782, + "mean_recall_at_4000_tokens": 0.8357, + "mean_recall_at_16000_tokens": 0.8495, + "mean_precision_at_10": 0.1454, + "zero_hit_rate": 0.0648, + "median_tokens_delivered": 3939, + "median_latency_ms": 757.492, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 100, + "mean_recall_at_5": 0.5715, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.7448, + "full_recall_at_10": 0.53, + "mean_recall_at_20": 0.8575, + "full_recall_at_20": 0.74, + "mean_recall_at_1000_tokens": 0.4665, + "mean_recall_at_4000_tokens": 0.8425, + "mean_recall_at_16000_tokens": 0.8575, + "mean_precision_at_10": 0.151, + "zero_hit_rate": 0.05, + "median_tokens_delivered": 3956.5, + "median_latency_ms": 757.492, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 8, + "mean_recall_at_5": 0.625, + "full_recall_at_5": 0.625, + "mean_recall_at_10": 0.75, + "full_recall_at_10": 0.75, + "mean_recall_at_20": 0.75, + "full_recall_at_20": 0.75, + "mean_recall_at_1000_tokens": 0.625, + "mean_recall_at_4000_tokens": 0.75, + "mean_recall_at_16000_tokens": 0.75, + "mean_precision_at_10": 0.075, + "zero_hit_rate": 0.25, + "median_tokens_delivered": 3769, + "median_latency_ms": 768.644, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.5784, + "full_recall_at_5": 0.2222, + "mean_recall_at_10": 0.7698, + "full_recall_at_10": 0.5926, + "mean_recall_at_20": 0.8673, + "full_recall_at_20": 0.7778, + "mean_recall_at_1000_tokens": 0.4765, + "mean_recall_at_4000_tokens": 0.8395, + "mean_recall_at_16000_tokens": 0.8673, + "mean_precision_at_10": 0.1556, + "zero_hit_rate": 0.0556, + "median_tokens_delivered": 3958.5, + "median_latency_ms": 773.423, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.5725, + "full_recall_at_5": 0.2407, + "mean_recall_at_10": 0.7207, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.8318, + "full_recall_at_20": 0.7037, + "mean_recall_at_1000_tokens": 0.4799, + "mean_recall_at_4000_tokens": 0.8318, + "mean_recall_at_16000_tokens": 0.8318, + "mean_precision_at_10": 0.1352, + "zero_hit_rate": 0.0741, + "median_tokens_delivered": 3926, + "median_latency_ms": 752.168, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 16253.7, + "peak_rss_mb": 18995.6, + "delta_rss_mb": 2741.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "jcodemunch", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.3642, + "full_recall_at_5": 0.1389, + "mean_recall_at_10": 0.3642, + "full_recall_at_10": 0.1389, + "mean_recall_at_20": 0.3642, + "full_recall_at_20": 0.1389, + "mean_recall_at_1000_tokens": 0.2461, + "mean_recall_at_4000_tokens": 0.3565, + "mean_recall_at_16000_tokens": 0.3642, + "mean_precision_at_10": 0.3123, + "zero_hit_rate": 0.3981, + "median_tokens_delivered": 1164.5, + "median_latency_ms": 574.1495, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 10 + }, + "baseline_missed": { + "cases": 100, + "mean_recall_at_5": 0.3333, + "full_recall_at_5": 0.09, + "mean_recall_at_10": 0.3333, + "full_recall_at_10": 0.09, + "mean_recall_at_20": 0.3333, + "full_recall_at_20": 0.09, + "mean_recall_at_1000_tokens": 0.2158, + "mean_recall_at_4000_tokens": 0.325, + "mean_recall_at_16000_tokens": 0.3333, + "mean_precision_at_10": 0.3123, + "zero_hit_rate": 0.41, + "median_tokens_delivered": 1232.5, + "median_latency_ms": 572.2695, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 10 + }, + "baseline_solved": { + "cases": 8, + "mean_recall_at_5": 0.75, + "full_recall_at_5": 0.75, + "mean_recall_at_10": 0.75, + "full_recall_at_10": 0.75, + "mean_recall_at_20": 0.75, + "full_recall_at_20": 0.75, + "mean_recall_at_1000_tokens": 0.625, + "mean_recall_at_4000_tokens": 0.75, + "mean_recall_at_16000_tokens": 0.75, + "mean_precision_at_10": 0.3125, + "zero_hit_rate": 0.25, + "median_tokens_delivered": 621, + "median_latency_ms": 584.301, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3611, + "full_recall_at_5": 0.1296, + "mean_recall_at_10": 0.3611, + "full_recall_at_10": 0.1296, + "mean_recall_at_20": 0.3611, + "full_recall_at_20": 0.1296, + "mean_recall_at_1000_tokens": 0.2161, + "mean_recall_at_4000_tokens": 0.3457, + "mean_recall_at_16000_tokens": 0.3611, + "mean_precision_at_10": 0.3333, + "zero_hit_rate": 0.4074, + "median_tokens_delivered": 1072.5, + "median_latency_ms": 562.0635, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 4 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3673, + "full_recall_at_5": 0.1481, + "mean_recall_at_10": 0.3673, + "full_recall_at_10": 0.1481, + "mean_recall_at_20": 0.3673, + "full_recall_at_20": 0.1481, + "mean_recall_at_1000_tokens": 0.2762, + "mean_recall_at_4000_tokens": 0.3673, + "mean_recall_at_16000_tokens": 0.3673, + "mean_precision_at_10": 0.2914, + "zero_hit_rate": 0.3889, + "median_tokens_delivered": 1232.5, + "median_latency_ms": 577.1745, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 6 + } + }, + "outcomes": { + "answered": 98, + "indexed-but-returned-nothing": 10 + }, + "resources": { + "baseline_rss_mb": 18647.9, + "peak_rss_mb": 21766.4, + "delta_rss_mb": 3118.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "cocoindex-code", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.0185, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0278, + "full_recall_at_10": 0.0093, + "mean_recall_at_20": 0.0278, + "full_recall_at_20": 0.0093, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0231, + "mean_recall_at_16000_tokens": 0.0278, + "mean_precision_at_10": 0.0073, + "zero_hit_rate": 0.9537, + "median_tokens_delivered": 2607, + "median_latency_ms": 597.5245, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 100, + "mean_recall_at_5": 0.02, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.03, + "full_recall_at_10": 0.01, + "mean_recall_at_20": 0.03, + "full_recall_at_20": 0.01, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.025, + "mean_recall_at_16000_tokens": 0.03, + "mean_precision_at_10": 0.0079, + "zero_hit_rate": 0.95, + "median_tokens_delivered": 2607, + "median_latency_ms": 597.5245, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 8, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 2607, + "median_latency_ms": 585.2905, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.0093, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0093, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0093, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0093, + "mean_recall_at_16000_tokens": 0.0093, + "mean_precision_at_10": 0.0031, + "zero_hit_rate": 0.9815, + "median_tokens_delivered": 2607, + "median_latency_ms": 565.0125, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.0278, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0463, + "full_recall_at_10": 0.0185, + "mean_recall_at_20": 0.0463, + "full_recall_at_20": 0.0185, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.037, + "mean_recall_at_16000_tokens": 0.0463, + "mean_precision_at_10": 0.0115, + "zero_hit_rate": 0.9259, + "median_tokens_delivered": 2607, + "median_latency_ms": 641.6225, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 21015.4, + "peak_rss_mb": 25562.4, + "delta_rss_mb": 4547, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.284, + "full_recall_at_5": 0.1204, + "mean_recall_at_10": 0.3957, + "full_recall_at_10": 0.213, + "mean_recall_at_20": 0.4577, + "full_recall_at_20": 0.2963, + "mean_recall_at_1000_tokens": 0.2887, + "mean_recall_at_4000_tokens": 0.4346, + "mean_recall_at_16000_tokens": 0.4577, + "mean_precision_at_10": 0.1039, + "zero_hit_rate": 0.3981, + "median_tokens_delivered": 1676.5, + "median_latency_ms": 958.7755, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 100, + "mean_recall_at_5": 0.2667, + "full_recall_at_5": 0.09, + "mean_recall_at_10": 0.3873, + "full_recall_at_10": 0.19, + "mean_recall_at_20": 0.4543, + "full_recall_at_20": 0.28, + "mean_recall_at_1000_tokens": 0.2718, + "mean_recall_at_4000_tokens": 0.4293, + "mean_recall_at_16000_tokens": 0.4543, + "mean_precision_at_10": 0.1072, + "zero_hit_rate": 0.39, + "median_tokens_delivered": 1676.5, + "median_latency_ms": 962.9855, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 8, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.5, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0.5, + "mean_recall_at_4000_tokens": 0.5, + "mean_recall_at_16000_tokens": 0.5, + "mean_precision_at_10": 0.0625, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 2044, + "median_latency_ms": 943.7645, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.2253, + "full_recall_at_5": 0.0741, + "mean_recall_at_10": 0.3407, + "full_recall_at_10": 0.1481, + "mean_recall_at_20": 0.4123, + "full_recall_at_20": 0.2593, + "mean_recall_at_1000_tokens": 0.292, + "mean_recall_at_4000_tokens": 0.4123, + "mean_recall_at_16000_tokens": 0.4123, + "mean_precision_at_10": 0.0852, + "zero_hit_rate": 0.4444, + "median_tokens_delivered": 1675, + "median_latency_ms": 958.7755, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3426, + "full_recall_at_5": 0.1667, + "mean_recall_at_10": 0.4506, + "full_recall_at_10": 0.2778, + "mean_recall_at_20": 0.5031, + "full_recall_at_20": 0.3333, + "mean_recall_at_1000_tokens": 0.2855, + "mean_recall_at_4000_tokens": 0.4568, + "mean_recall_at_16000_tokens": 0.5031, + "mean_precision_at_10": 0.1225, + "zero_hit_rate": 0.3519, + "median_tokens_delivered": 1679, + "median_latency_ms": 958.2645, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 20197, + "peak_rss_mb": 21761.7, + "delta_rss_mb": 1564.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 84590592 + } + }, + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.5227, + "full_recall_at_5": 0.2778, + "mean_recall_at_10": 0.6938, + "full_recall_at_10": 0.5278, + "mean_recall_at_20": 0.8086, + "full_recall_at_20": 0.6944, + "mean_recall_at_1000_tokens": 0.7941, + "mean_recall_at_4000_tokens": 0.8086, + "mean_recall_at_16000_tokens": 0.8086, + "mean_precision_at_10": 0.1412, + "zero_hit_rate": 0.0833, + "median_tokens_delivered": 927, + "median_latency_ms": 139.0135, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 100, + "mean_recall_at_5": 0.5045, + "full_recall_at_5": 0.24, + "mean_recall_at_10": 0.6893, + "full_recall_at_10": 0.51, + "mean_recall_at_20": 0.8133, + "full_recall_at_20": 0.69, + "mean_recall_at_1000_tokens": 0.7977, + "mean_recall_at_4000_tokens": 0.8133, + "mean_recall_at_16000_tokens": 0.8133, + "mean_precision_at_10": 0.1465, + "zero_hit_rate": 0.07, + "median_tokens_delivered": 937.5, + "median_latency_ms": 139.0135, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 8, + "mean_recall_at_5": 0.75, + "full_recall_at_5": 0.75, + "mean_recall_at_10": 0.75, + "full_recall_at_10": 0.75, + "mean_recall_at_20": 0.75, + "full_recall_at_20": 0.75, + "mean_recall_at_1000_tokens": 0.75, + "mean_recall_at_4000_tokens": 0.75, + "mean_recall_at_16000_tokens": 0.75, + "mean_precision_at_10": 0.075, + "zero_hit_rate": 0.25, + "median_tokens_delivered": 718, + "median_latency_ms": 139.144, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.513, + "full_recall_at_5": 0.2593, + "mean_recall_at_10": 0.7148, + "full_recall_at_10": 0.5185, + "mean_recall_at_20": 0.8488, + "full_recall_at_20": 0.7222, + "mean_recall_at_1000_tokens": 0.8198, + "mean_recall_at_4000_tokens": 0.8488, + "mean_recall_at_16000_tokens": 0.8488, + "mean_precision_at_10": 0.1431, + "zero_hit_rate": 0.037, + "median_tokens_delivered": 945, + "median_latency_ms": 140.7625, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.5324, + "full_recall_at_5": 0.2963, + "mean_recall_at_10": 0.6728, + "full_recall_at_10": 0.537, + "mean_recall_at_20": 0.7685, + "full_recall_at_20": 0.6667, + "mean_recall_at_1000_tokens": 0.7685, + "mean_recall_at_4000_tokens": 0.7685, + "mean_recall_at_16000_tokens": 0.7685, + "mean_precision_at_10": 0.1393, + "zero_hit_rate": 0.1296, + "median_tokens_delivered": 902.5, + "median_latency_ms": 137.913, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 20025, + "peak_rss_mb": 20239.8, + "delta_rss_mb": 214.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 534724608 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "test/create.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 17695, + "latency_ms": 18.89, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29330, + "latency_ms": 12.479, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44537, + "latency_ms": 12.238, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 30339, + "latency_ms": 12.791, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28268, + "latency_ms": 12.045, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 37092, + "latency_ms": 12.878, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24761, + "latency_ms": 12.596, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 32998, + "latency_ms": 12.588, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 45500, + "latency_ms": 11.859, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 40700, + "latency_ms": 12.018, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 63090, + "latency_ms": 11.899, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29834, + "latency_ms": 11.563, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 19371, + "latency_ms": 11.105, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26240, + "latency_ms": 12.1, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 41881, + "latency_ms": 14.772, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40853, + "latency_ms": 14.061, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 41220, + "latency_ms": 11.996, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 19500, + "latency_ms": 11.701, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 40496, + "latency_ms": 12.91, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18203, + "latency_ms": 11.71, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24438, + "latency_ms": 11.939, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20036, + "latency_ms": 12.086, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20481, + "latency_ms": 12.204, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 27897, + "latency_ms": 12.021, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 48003, + "latency_ms": 11.182, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "tsconfig.json" + ], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 44800, + "latency_ms": 11.669, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25732, + "latency_ms": 11.284, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19504, + "latency_ms": 11.096, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29947, + "latency_ms": 11.251, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17080, + "latency_ms": 10.92, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 42253, + "latency_ms": 10.631, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 50293, + "latency_ms": 11.901, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 17, + "tokens_delivered": 16939, + "latency_ms": 11.771, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 64594, + "latency_ms": 11.082, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [ + "source/as-promise/types.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 20438, + "latency_ms": 12.393, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19771, + "latency_ms": 11.057, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32900, + "latency_ms": 11.346, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 36724, + "latency_ms": 14.628, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [ + "test/agent.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 29961, + "latency_ms": 13.075, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29680, + "latency_ms": 10.786, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 45651, + "latency_ms": 10.424, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 20651, + "latency_ms": 10.169, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 26908, + "latency_ms": 11.596, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14053, + "latency_ms": 10.516, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts" + ], + "missed": [ + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 23093, + "latency_ms": 11.298, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 34438, + "latency_ms": 10.4, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 40983, + "latency_ms": 10.872, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 7, + "tokens_delivered": 38400, + "latency_ms": 9.852, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 56327, + "latency_ms": 9.669, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/error.ts", + "test/helpers.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 2, + "tokens_delivered": 27556, + "latency_ms": 9.323, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31766, + "latency_ms": 10.968, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 9, + "tokens_delivered": 25946, + "latency_ms": 10.454, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 61131, + "latency_ms": 10.91, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43051, + "latency_ms": 9.855, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 55773, + "latency_ms": 10.827, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43907, + "latency_ms": 10.936, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 26684, + "latency_ms": 11.709, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 28696, + "latency_ms": 10.059, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 29633, + "latency_ms": 11.337, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31568, + "latency_ms": 9.61, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 32483, + "latency_ms": 9.831, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 16998, + "latency_ms": 9.85, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 28802, + "latency_ms": 13.924, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18220, + "latency_ms": 12.477, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 34057, + "latency_ms": 10.7, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27444, + "latency_ms": 10.626, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [ + "test/progress.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 43111, + "latency_ms": 10.25, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 36388, + "latency_ms": 11.364, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 26052, + "latency_ms": 10.349, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34828, + "latency_ms": 10.715, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 47799, + "latency_ms": 11.577, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15060, + "latency_ms": 10.983, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35728, + "latency_ms": 10.148, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30113, + "latency_ms": 9.878, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 26343, + "latency_ms": 11.734, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [ + "test/early-307-redirects.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 66990, + "latency_ms": 10.999, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 33984, + "latency_ms": 11.101, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 70809, + "latency_ms": 11.686, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16148, + "latency_ms": 11.093, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22493, + "latency_ms": 10.096, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43364, + "latency_ms": 10.409, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25107, + "latency_ms": 9.385, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 34784, + "latency_ms": 9.996, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21815, + "latency_ms": 10.95, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 39157, + "latency_ms": 10.71, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21610, + "latency_ms": 10.254, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 36075, + "latency_ms": 10.271, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24832, + "latency_ms": 12.542, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/options.ts", + "test/headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 32152, + "latency_ms": 12.87, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 25999, + "latency_ms": 11.102, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16382, + "latency_ms": 10.993, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37626, + "latency_ms": 10.505, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 57177, + "latency_ms": 10.626, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 34640, + "latency_ms": 10.718, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24940, + "latency_ms": 10.315, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 39937, + "latency_ms": 11.771, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 42625, + "latency_ms": 10.683, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 30034, + "latency_ms": 10.81, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "source/core/index.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 26779, + "latency_ms": 10.74, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "test/retry.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 40775, + "latency_ms": 10.748, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13037, + "latency_ms": 10.647, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23583, + "latency_ms": 11.288, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27860, + "latency_ms": 10.299, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24606, + "latency_ms": 11.034, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28939, + "latency_ms": 11.034, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "test/extend.types.ts" + ], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 11, + "tokens_delivered": 24896, + "latency_ms": 11.766, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 56437, + "latency_ms": 11.818, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17646, + "latency_ms": 10.194, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 58679, + "latency_ms": 31.298, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 89852, + "latency_ms": 23.721, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 82176, + "latency_ms": 23.642, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 57240, + "latency_ms": 26.144, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73871, + "latency_ms": 22.405, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/gzip.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 162539, + "latency_ms": 21.473, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 87852, + "latency_ms": 23.214, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 73650, + "latency_ms": 24.318, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 103569, + "latency_ms": 23.324, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 71989, + "latency_ms": 23.847, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 162719, + "latency_ms": 25.043, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55161, + "latency_ms": 30.759, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 49756, + "latency_ms": 26.832, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 69499, + "latency_ms": 25.638, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 57290, + "latency_ms": 25.826, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 93628, + "latency_ms": 24.363, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 57286, + "latency_ms": 25.208, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71767, + "latency_ms": 24.515, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 91285, + "latency_ms": 22.47, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 77785, + "latency_ms": 25.581, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 88524, + "latency_ms": 34.772, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 53380, + "latency_ms": 24.291, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 91140, + "latency_ms": 26.306, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 76834, + "latency_ms": 29.045, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 101370, + "latency_ms": 25.174, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 76435, + "latency_ms": 24.599, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 76435, + "latency_ms": 23.637, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 87924, + "latency_ms": 23.27, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 76166, + "latency_ms": 30.56, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 68859, + "latency_ms": 29.037, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 71393, + "latency_ms": 22.2, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 70103, + "latency_ms": 27.378, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 51379, + "latency_ms": 29.022, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 103622, + "latency_ms": 25.157, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70281, + "latency_ms": 26.352, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [ + "source/types.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 103976, + "latency_ms": 23.998, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 87647, + "latency_ms": 23.968, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54612, + "latency_ms": 25.847, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 50065, + "latency_ms": 25.202, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 163314, + "latency_ms": 25.119, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 72182, + "latency_ms": 24.585, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 89284, + "latency_ms": 24.626, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 103622, + "latency_ms": 23.414, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 57236, + "latency_ms": 42.499, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 56354, + "latency_ms": 24.872, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 77422, + "latency_ms": 24.018, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 84288, + "latency_ms": 24.455, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 89510, + "latency_ms": 24.791, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 90189, + "latency_ms": 22.52, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 57196, + "latency_ms": 26.465, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 82777, + "latency_ms": 23.798, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts" + ], + "missed": [ + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 55092, + "latency_ms": 25.373, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 162744, + "latency_ms": 23.469, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 55701, + "latency_ms": 39.12, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 82287, + "latency_ms": 24.473, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/utils/get-body-size.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 69971, + "latency_ms": 25.146, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 49545, + "latency_ms": 25.444, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 54988, + "latency_ms": 25.576, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 51355, + "latency_ms": 24.478, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 69534, + "latency_ms": 24.867, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 55421, + "latency_ms": 24.76, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 55679, + "latency_ms": 26.001, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 51051, + "latency_ms": 28.017, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82219, + "latency_ms": 24.357, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 54539, + "latency_ms": 28.514, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 94625, + "latency_ms": 23.45, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73537, + "latency_ms": 37.772, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 71935, + "latency_ms": 23.553, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 58746, + "latency_ms": 23.576, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82168, + "latency_ms": 23.547, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 57062, + "latency_ms": 24.562, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 87789, + "latency_ms": 28.237, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "test/post.ts" + ], + "missed": [ + "source/core/utils/get-body-size.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 88149, + "latency_ms": 25.349, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 54889, + "latency_ms": 25.389, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 56081, + "latency_ms": 30.684, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [ + "test/early-307-redirects.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 163538, + "latency_ms": 25.811, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 77420, + "latency_ms": 30.22, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 161120, + "latency_ms": 34.498, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 77692, + "latency_ms": 24.426, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 73848, + "latency_ms": 24.299, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 95064, + "latency_ms": 24.496, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 82287, + "latency_ms": 24.87, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 77134, + "latency_ms": 24.072, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 79062, + "latency_ms": 25.588, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 69515, + "latency_ms": 31.573, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 72804, + "latency_ms": 28.412, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82400, + "latency_ms": 25.886, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 81765, + "latency_ms": 24.04, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 72116, + "latency_ms": 24.949, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 53420, + "latency_ms": 41.037, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 55889, + "latency_ms": 24.164, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 82176, + "latency_ms": 22.847, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82513, + "latency_ms": 25.281, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 82131, + "latency_ms": 24.802, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 57497, + "latency_ms": 27.627, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 90031, + "latency_ms": 25.869, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 72711, + "latency_ms": 25.352, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 56960, + "latency_ms": 24.999, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 93102, + "latency_ms": 21.589, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 86056, + "latency_ms": 21.472, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 74001, + "latency_ms": 22.518, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 73071, + "latency_ms": 36.274, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 97394, + "latency_ms": 23.352, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82168, + "latency_ms": 23.395, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "source/create.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 82112, + "latency_ms": 23.902, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "test/create.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "missed": [ + "source/types.ts", + "test/extend.types.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 103627, + "latency_ms": 26.01, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 87367, + "latency_ms": 28.657, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 70574, + "latency_ms": 27.514, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3889, + "latency_ms": 491.834, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3983, + "latency_ms": 570.579, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3730, + "latency_ms": 568.376, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 4085, + "latency_ms": 528.892, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3990, + "latency_ms": 557.184, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/http.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4053, + "latency_ms": 746.235, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4247, + "latency_ms": 579.381, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4063, + "latency_ms": 516.017, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3912, + "latency_ms": 638.973, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3664, + "latency_ms": 685.546, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3786, + "latency_ms": 896.153, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3480, + "latency_ms": 646.316, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/normalize-arguments.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 3961, + "latency_ms": 620.979, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3884, + "latency_ms": 732.78, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3809, + "latency_ms": 737.184, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4921, + "latency_ms": 821.154, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4037, + "latency_ms": 718.309, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3956, + "latency_ms": 743.079, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4208, + "latency_ms": 769.626, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3761, + "latency_ms": 711.073, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3490, + "latency_ms": 859.238, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4119, + "latency_ms": 639.179, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3737, + "latency_ms": 835.507, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3516, + "latency_ms": 785.747, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4334, + "latency_ms": 885.978, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3709, + "latency_ms": 778.336, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3866, + "latency_ms": 836.231, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4272, + "latency_ms": 811.779, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3946, + "latency_ms": 725.702, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3819, + "latency_ms": 715.595, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.75, + "precision_at_5": 0.6, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 4152, + "latency_ms": 796.933, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4925, + "latency_ms": 715.075, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3928, + "latency_ms": 668.149, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3878, + "latency_ms": 933.384, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4437, + "latency_ms": 765.841, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4575, + "latency_ms": 836.005, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4277, + "latency_ms": 788.849, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3938, + "latency_ms": 691.013, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 3482, + "latency_ms": 697.741, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3775, + "latency_ms": 1075.964, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3900, + "latency_ms": 1215.244, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4630, + "latency_ms": 826.99, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4057, + "latency_ms": 886.802, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3672, + "latency_ms": 650.588, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2143, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3731, + "latency_ms": 659.343, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3967, + "latency_ms": 855.336, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3768, + "latency_ms": 886.585, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4119, + "latency_ms": 832.073, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4505, + "latency_ms": 760.071, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts" + ], + "missed": [ + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.6, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 4, + "tokens_delivered": 4048, + "latency_ms": 738.666, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3746, + "latency_ms": 725.834, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 3940, + "latency_ms": 730.974, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3981, + "latency_ms": 974.956, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3506, + "latency_ms": 775.342, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 4344, + "latency_ms": 907.738, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3577, + "latency_ms": 726.952, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4030, + "latency_ms": 619.757, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3840, + "latency_ms": 660.487, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3828, + "latency_ms": 711.89, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3648, + "latency_ms": 681.558, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3409, + "latency_ms": 665.746, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 4164, + "latency_ms": 634.027, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3938, + "latency_ms": 620.049, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3643, + "latency_ms": 865.036, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3925, + "latency_ms": 669.991, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4073, + "latency_ms": 844.662, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3567, + "latency_ms": 751.541, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3832, + "latency_ms": 677.033, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3655, + "latency_ms": 800.462, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3888, + "latency_ms": 738.577, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4102, + "latency_ms": 727.856, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3702, + "latency_ms": 753.466, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3880, + "latency_ms": 754.913, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4377, + "latency_ms": 657.114, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3979, + "latency_ms": 609.831, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2308, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3847, + "latency_ms": 1078.51, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3793, + "latency_ms": 750.87, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4094, + "latency_ms": 975.595, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4403, + "latency_ms": 730.232, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3494, + "latency_ms": 683.534, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4615, + "latency_ms": 893.865, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4071, + "latency_ms": 1181.017, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3531, + "latency_ms": 976.741, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4418, + "latency_ms": 901.129, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4151, + "latency_ms": 801.826, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4262, + "latency_ms": 776.698, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3781, + "latency_ms": 795.849, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3937, + "latency_ms": 788.738, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 3989, + "latency_ms": 871.807, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3803, + "latency_ms": 699.657, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4047, + "latency_ms": 668.293, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4768, + "latency_ms": 767.816, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4361, + "latency_ms": 837.367, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4241, + "latency_ms": 774.416, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3890, + "latency_ms": 749.085, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 4162, + "latency_ms": 834.846, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3957, + "latency_ms": 753.832, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 3745, + "latency_ms": 717.153, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 4249, + "latency_ms": 849.988, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3841, + "latency_ms": 866.866, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3516, + "latency_ms": 887.607, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3855, + "latency_ms": 855.736, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4196, + "latency_ms": 876.345, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3714, + "latency_ms": 807.066, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "source/create.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 4026, + "latency_ms": 988.926, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/extend.types.ts" + ], + "missed": [ + "test/create.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 1, + "tokens_delivered": 3612, + "latency_ms": 905.374, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4150, + "latency_ms": 868.153, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 4113, + "latency_ms": 777.22, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 667, + "latency_ms": 467.434, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 554, + "latency_ms": 467.602, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1389, + "latency_ms": 789.041, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 446, + "latency_ms": 443.717, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 730, + "latency_ms": 487.348, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3651, + "latency_ms": 565.288, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1737, + "latency_ms": 539.179, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1629, + "latency_ms": 486.975, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1273, + "latency_ms": 578.775, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 459, + "latency_ms": 473.293, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1192, + "latency_ms": 558.374, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1110, + "latency_ms": 527.436, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 521, + "latency_ms": 468.076, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 925, + "latency_ms": 488.331, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2120, + "latency_ms": 528.737, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "test/retry.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2427, + "latency_ms": 631.267, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1912, + "latency_ms": 637.513, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 471, + "latency_ms": 586.55, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 704, + "latency_ms": 531.628, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6068, + "latency_ms": 575.574, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 458, + "latency_ms": 602.741, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2796, + "latency_ms": 514.977, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2569, + "latency_ms": 573.726, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 969, + "latency_ms": 532.869, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1359, + "latency_ms": 576.354, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 589, + "latency_ms": 539.395, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1551, + "latency_ms": 617.814, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1170, + "latency_ms": 558.775, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1553, + "latency_ms": 638.677, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1922, + "latency_ms": 668.889, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6667, + "recall_at_10": 0.5, + "precision_at_10": 0.6667, + "recall_at_20": 0.5, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1878, + "latency_ms": 754.651, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3996, + "latency_ms": 722.316, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 468, + "latency_ms": 1156.547, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 2785, + "latency_ms": 696.92, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2184, + "latency_ms": 568.166, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 2212, + "latency_ms": 741.259, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5024, + "latency_ms": 676.104, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3092, + "latency_ms": 616.549, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.3333, + "recall_at_10": 0.25, + "precision_at_10": 0.3333, + "recall_at_20": 0.25, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 601, + "latency_ms": 533.942, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 484, + "latency_ms": 857.331, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2188, + "latency_ms": 656.17, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3574, + "latency_ms": 849.217, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 459, + "latency_ms": 721.528, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 566, + "latency_ms": 541.164, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 2095, + "latency_ms": 702.392, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2872, + "latency_ms": 742.72, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2209, + "latency_ms": 654.612, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5802, + "latency_ms": 1089.345, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 3715, + "latency_ms": 538.068, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 443, + "latency_ms": 431.755, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2219, + "latency_ms": 380.672, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.3333, + "recall_at_10": 0.25, + "precision_at_10": 0.3333, + "recall_at_20": 0.25, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 1118, + "latency_ms": 408.902, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 660, + "latency_ms": 473.736, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 551, + "latency_ms": 646.559, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.5, + "recall_at_10": 0.6667, + "precision_at_10": 0.5, + "recall_at_20": 0.6667, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 540, + "latency_ms": 431.148, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 944, + "latency_ms": 361.562, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 520, + "latency_ms": 462.29, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 420, + "latency_ms": 486.816, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1925, + "latency_ms": 545.204, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 530, + "latency_ms": 381.148, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4155, + "latency_ms": 434.054, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 3597, + "latency_ms": 486.898, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 599, + "latency_ms": 367.786, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 635, + "latency_ms": 441.45, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1875, + "latency_ms": 460.61, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 4086, + "latency_ms": 453.545, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 513, + "latency_ms": 464.478, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 437, + "latency_ms": 491.519, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 424, + "latency_ms": 455.609, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 534, + "latency_ms": 433.727, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 561, + "latency_ms": 488.622, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1018, + "latency_ms": 493.531, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2501, + "latency_ms": 514.629, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 688, + "latency_ms": 462.091, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 735, + "latency_ms": 484.822, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6383, + "latency_ms": 626.066, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 729, + "latency_ms": 594.029, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 496, + "latency_ms": 730.861, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1114, + "latency_ms": 702.183, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105, + "latency_ms": 595.918, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4493, + "latency_ms": 598.617, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5126, + "latency_ms": 631.913, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 421, + "latency_ms": 619.333, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2079, + "latency_ms": 618.55, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1816, + "latency_ms": 631.658, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 858, + "latency_ms": 829.151, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3126, + "latency_ms": 677.472, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 489, + "latency_ms": 568.675, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 459, + "latency_ms": 524.633, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2005, + "latency_ms": 523.898, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2722, + "latency_ms": 545.754, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3810, + "latency_ms": 703.999, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6923, + "latency_ms": 555.452, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2676, + "latency_ms": 594.207, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2179, + "latency_ms": 655.89, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5703, + "latency_ms": 643.414, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 550, + "latency_ms": 570.813, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 448, + "latency_ms": 574.573, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 402, + "latency_ms": 826.097, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1035, + "latency_ms": 698.472, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 504, + "latency_ms": 673.686, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 453, + "latency_ms": 635.131, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1159, + "latency_ms": 695.185, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 541, + "latency_ms": 740.991, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 623, + "latency_ms": 696.301, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7435, + "latency_ms": 727.279, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2430, + "latency_ms": 715.549, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2430, + "latency_ms": 618.373, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 5071, + "latency_ms": 9181.794, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4995, + "latency_ms": 15264.59, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5036, + "latency_ms": 121219.723, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4271, + "latency_ms": 121185.32, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4678, + "latency_ms": 22776.851, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4552, + "latency_ms": 82582.279, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5238, + "latency_ms": 22195.197, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 5244, + "latency_ms": 2674.882, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 5028, + "latency_ms": 20896.72, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 4488, + "latency_ms": 11639.315, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 5049, + "latency_ms": 19907.336, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 4617.335, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 759.323, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 899.085, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 921.172, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 933.891, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 971.886, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1184.33, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 773.044, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 790.971, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 612.741, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 606.868, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 705.104, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 742.7, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 820.181, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 778.921, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 833.946, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 748.71, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 598.487, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 873.725, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 812.233, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 700.793, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 782.318, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 974.807, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 727.486, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 830.4, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 815.173, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 971.374, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 825.223, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 649.736, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 886.915, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 644.619, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 710.576, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 972.89, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 833.532, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1018.268, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 737.414, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 549.731, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 560.782, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 582.555, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 573.43, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 735.299, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 657.311, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 602.132, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 568.377, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 523.842, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 514.552, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 563.224, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 528.181, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 531.883, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 630.938, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 517.911, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 883.244, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 524.478, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 502.885, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 508.574, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 571.678, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 598.903, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 566.801, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 500.07, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 527.711, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 522.479, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 505.972, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 391.086, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 596.562, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 578.861, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 551.076, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 527.662, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 495.158, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 537.459, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 540.221, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 475.785, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 514.207, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 501.616, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 506.636, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 496.933, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 504.339, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 504.97, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 638.626, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 566.97, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 512.174, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 504.761, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 529.114, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 514.816, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 515.605, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 514.212, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 515.951, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 520.696, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 493.745, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 486.56, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 395.865, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 482.431, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 393.608, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 481.877, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 387.373, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 495.649, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 382.1, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 410.565, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 15, + "tokens_delivered": 6858, + "latency_ms": 812.978, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1656, + "latency_ms": 1075.484, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3335, + "latency_ms": 1128.856, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2594, + "latency_ms": 875.109, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 5791, + "latency_ms": 1051.794, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 2, + "tokens_delivered": 1676, + "latency_ms": 1252.642, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4439, + "latency_ms": 1032.486, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1656, + "latency_ms": 1080.179, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 8203, + "latency_ms": 1073.976, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2927, + "latency_ms": 1036.974, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 6443, + "latency_ms": 1307.697, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2733, + "latency_ms": 868.808, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 8, + "tokens_delivered": 6563, + "latency_ms": 769.154, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1667, + "latency_ms": 908.513, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1695, + "latency_ms": 881.741, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1686, + "latency_ms": 1068.232, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 165, + "latency_ms": 954.175, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 160, + "latency_ms": 942.578, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1661, + "latency_ms": 1030.029, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 491, + "latency_ms": 1570.545, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3555, + "latency_ms": 1173.989, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3960, + "latency_ms": 808.18, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2833, + "latency_ms": 1151.316, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3776, + "latency_ms": 966.149, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1673, + "latency_ms": 1129.51, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1664, + "latency_ms": 1060.257, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1663, + "latency_ms": 1065.784, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1676, + "latency_ms": 983.274, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1662, + "latency_ms": 990.879, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1659, + "latency_ms": 858.297, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 4937, + "latency_ms": 903.517, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 3653, + "latency_ms": 815.441, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2247, + "latency_ms": 755.727, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 1667, + "latency_ms": 1054.008, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1680, + "latency_ms": 815.761, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 1677, + "latency_ms": 1061.275, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1672, + "latency_ms": 971.26, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1663, + "latency_ms": 809.405, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 521, + "latency_ms": 750.114, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 5830, + "latency_ms": 1169.155, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1668, + "latency_ms": 893.541, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1715, + "latency_ms": 978.71, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/utils/timer.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 12299, + "latency_ms": 1051.893, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1450, + "latency_ms": 772.695, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3436, + "latency_ms": 774.601, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 894, + "latency_ms": 962.354, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3247, + "latency_ms": 970.04, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1661, + "latency_ms": 978.123, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 12, + "tokens_delivered": 1680, + "latency_ms": 982.084, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4373, + "latency_ms": 835.186, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1679, + "latency_ms": 979.909, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 6, + "tokens_delivered": 7242, + "latency_ms": 772.616, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1142, + "latency_ms": 1160.128, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1668, + "latency_ms": 826.11, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1661, + "latency_ms": 954.862, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5707, + "latency_ms": 797.302, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1414, + "latency_ms": 737.237, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2113, + "latency_ms": 834.536, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1343, + "latency_ms": 750.164, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1666, + "latency_ms": 791.647, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 154, + "latency_ms": 764.746, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 2407, + "latency_ms": 774.813, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3863, + "latency_ms": 755.86, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3437, + "latency_ms": 959.758, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 1666, + "latency_ms": 810.339, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3007, + "latency_ms": 988.354, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2755, + "latency_ms": 933.873, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2834, + "latency_ms": 882.306, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 5412, + "latency_ms": 766.404, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1673, + "latency_ms": 954.021, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1653, + "latency_ms": 835.64, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1676, + "latency_ms": 1004.353, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 5149, + "latency_ms": 977.697, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1672, + "latency_ms": 822.615, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1314, + "latency_ms": 769.789, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1675, + "latency_ms": 1170.101, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1658, + "latency_ms": 953.656, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1667, + "latency_ms": 1151.622, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1656, + "latency_ms": 953.016, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2650, + "latency_ms": 921.004, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3589, + "latency_ms": 986.777, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1668, + "latency_ms": 945.904, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1667, + "latency_ms": 960.497, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 1661, + "latency_ms": 950.677, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4701, + "latency_ms": 811.384, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 230, + "latency_ms": 912.8, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1663, + "latency_ms": 970.679, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1669, + "latency_ms": 1000.26, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 4189, + "latency_ms": 998.587, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1679, + "latency_ms": 1097.663, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1667, + "latency_ms": 780.667, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1690, + "latency_ms": 973.957, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1675, + "latency_ms": 974.188, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2832, + "latency_ms": 968.192, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1312, + "latency_ms": 769.041, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 1679, + "latency_ms": 981.292, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1664, + "latency_ms": 899.632, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2420, + "latency_ms": 758.145, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 1677, + "latency_ms": 968.749, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 221, + "latency_ms": 967.123, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3593, + "latency_ms": 929.789, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2724, + "latency_ms": 913.522, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1669, + "latency_ms": 1004.666, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4150, + "latency_ms": 967.549, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 160, + "latency_ms": 957.793, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "missed": [ + "test/create.ts", + "test/extend.types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 8, + "tokens_delivered": 1656, + "latency_ms": 1042.953, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1247, + "latency_ms": 966.213, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 1682, + "latency_ms": 802.737, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 849, + "latency_ms": 119.56, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1264, + "latency_ms": 150.992, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 740, + "latency_ms": 142.267, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 855, + "latency_ms": 126.332, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 19, + "tokens_delivered": 822, + "latency_ms": 137.496, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1159, + "latency_ms": 188.889, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1518, + "latency_ms": 149.153, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 722, + "latency_ms": 131.785, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 931, + "latency_ms": 160.703, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 665, + "latency_ms": 130.689, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1250, + "latency_ms": 181.157, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 1937, + "latency_ms": 124.368, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 973, + "latency_ms": 112.257, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 891, + "latency_ms": 124.965, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 953, + "latency_ms": 138.541, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 916, + "latency_ms": 150.662, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1000, + "latency_ms": 122.876, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 858, + "latency_ms": 126.476, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 971, + "latency_ms": 148.065, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1387, + "latency_ms": 139.73, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 542, + "latency_ms": 147.25, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 979, + "latency_ms": 117.236, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 1131, + "latency_ms": 148.47, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1874, + "latency_ms": 146.682, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 488, + "latency_ms": 151.795, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1480, + "latency_ms": 139.486, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1052, + "latency_ms": 143.846, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 519, + "latency_ms": 149.084, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1062, + "latency_ms": 145.739, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 727, + "latency_ms": 128.059, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/response.ts" + ], + "missed": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 69, + "latency_ms": 133.02, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 1083, + "latency_ms": 126.108, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 733, + "latency_ms": 119.799, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 870, + "latency_ms": 167.57, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 914, + "latency_ms": 124.379, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [ + "source/types.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 858, + "latency_ms": 165.323, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 797, + "latency_ms": 153.744, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 683, + "latency_ms": 122.224, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [ + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 554, + "latency_ms": 111.624, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 822, + "latency_ms": 185.895, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 321, + "latency_ms": 135.708, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1082, + "latency_ms": 148.054, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 679, + "latency_ms": 156.768, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 432, + "latency_ms": 115.79, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 211, + "latency_ms": 118.633, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 475, + "latency_ms": 138.158, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 785, + "latency_ms": 144.166, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 803, + "latency_ms": 144.951, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1779, + "latency_ms": 145.468, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.4, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 7, + "tokens_delivered": 777, + "latency_ms": 121.629, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1114, + "latency_ms": 144.06, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 790, + "latency_ms": 118.167, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 979, + "latency_ms": 184.917, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 736, + "latency_ms": 120.704, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1396, + "latency_ms": 142.704, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 414, + "latency_ms": 121.258, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 1045, + "latency_ms": 114.117, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 835, + "latency_ms": 120.822, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1125, + "latency_ms": 109.493, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 957, + "latency_ms": 119.375, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 962, + "latency_ms": 110.986, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 1157, + "latency_ms": 118.442, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1196, + "latency_ms": 110.265, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1082, + "latency_ms": 140.649, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1064, + "latency_ms": 115.696, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 745, + "latency_ms": 150.093, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2046, + "latency_ms": 145.809, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 680, + "latency_ms": 141.517, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1418, + "latency_ms": 117.679, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 881, + "latency_ms": 140.469, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 866, + "latency_ms": 122.785, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1087, + "latency_ms": 145.998, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 802, + "latency_ms": 144.945, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1152, + "latency_ms": 122.155, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 937, + "latency_ms": 115.114, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1518, + "latency_ms": 189.128, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 821, + "latency_ms": 137.668, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1428, + "latency_ms": 186.901, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 923, + "latency_ms": 136.833, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 248, + "latency_ms": 135.779, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1171, + "latency_ms": 154.654, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 938, + "latency_ms": 142.849, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 736, + "latency_ms": 140.13, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 876, + "latency_ms": 137.826, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 676, + "latency_ms": 124.985, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1039, + "latency_ms": 136.8, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 1125, + "latency_ms": 141.049, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1202, + "latency_ms": 145.968, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 800, + "latency_ms": 134.974, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 891, + "latency_ms": 115.731, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1021, + "latency_ms": 122.949, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1062, + "latency_ms": 145.084, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1020, + "latency_ms": 143.723, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 56, + "latency_ms": 138.532, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 862, + "latency_ms": 121.185, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1050, + "latency_ms": 156.694, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1033, + "latency_ms": 141.014, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 700, + "latency_ms": 118.567, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 898, + "latency_ms": 145.06, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1410, + "latency_ms": 143.877, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 621, + "latency_ms": 135.277, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1068, + "latency_ms": 132.571, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 950, + "latency_ms": 149.525, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 909, + "latency_ms": 143.853, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 753, + "latency_ms": 140.876, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts" + ], + "missed": [ + "source/types.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 5, + "tokens_delivered": 1558, + "latency_ms": 159.635, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 959, + "latency_ms": 147.05, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 1084, + "latency_ms": 123.123, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/CV-fastify.json b/benchmarks/context-retrieval/results/CV-fastify.json new file mode 100644 index 00000000..8eb34f16 --- /dev/null +++ b/benchmarks/context-retrieval/results/CV-fastify.json @@ -0,0 +1,7317 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "fastify", + "head": "83e69762aff047940b376fc2a266d143499ef727" + }, + "tier": "medium", + "tier_code_files": 300, + "corpus_counts": { + "cases": 82, + "path_leak": 47, + "baseline_blind": 35, + "multi_file": 66 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.1762, + "control": 0, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "replace AssertionError with FST_ERR_PLUGIN_DEPENDENCY_NOT_REGISTERED in checkDependencies (#6774)", + "revision": "27f4d6222dd945238e96c1fdaa632d5dc3c37b2c", + "returned": [ + ".github/workflows/coverage-nix.yml", + "examples/route-prefix.js", + "test/internals/hooks.test.js", + "lib/content-type.js", + "test/router-options.test.js" + ] + }, + { + "provider_id": "churn-ranked", + "query": "add http method override warning (#6879)", + "revision": "b93d611c7cefc2cd4a0b6f05cc864f2b6c397a6d", + "returned": [ + "package.json", + "fastify.js", + ".github/workflows/ci.yml", + "lib/reply.js", + ".github/workflows/links-check.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "use JSON.stringify in onBadUrl for proper escaping (#6420)", + "revision": "970c575832521fff01cc018d928d84454811173b", + "returned": [ + "package.json", + "fastify.js", + ".github/workflows/ci.yml", + "lib/reply.js", + "lib/route.js" + ] + }, + { + "provider_id": "churn-ranked", + "query": "run preClose hook exactly once per declaring instance (#6940)", + "revision": "9a1535bcfe2a780cc6f9c6555df86e5072ca9a29", + "returned": [ + "package.json", + "fastify.js", + ".github/workflows/ci.yml", + "lib/reply.js", + "fastify.d.ts" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "uncatchable throws in writeHead when using async hook (#6881)", + "revision": "b6cdc6a40598d15f38fe7d399eb17f4b84b7bf06", + "returned": [ + "docs/Reference/Errors.md", + "examples/asyncawait.js", + "fastify.js", + "lib/errors.js", + "lib/four-oh-four.js" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "clear `[kState].readyPromise` for garbage collection (#6030)", + "revision": "7868e47194bcfb34cab2e51ea36bf10f764b538e", + "returned": [ + "test/hooks.test.js", + "fastify.js" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0024, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0329, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0091, + "mean_precision_at_10": 0.0012, + "zero_hit_rate": 0.9146, + "median_tokens_delivered": 30284.5, + "median_latency_ms": 17.098, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0024, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0329, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0091, + "mean_precision_at_10": 0.0012, + "zero_hit_rate": 0.9146, + "median_tokens_delivered": 30284.5, + "median_latency_ms": 17.098, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0043, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0255, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0053, + "mean_precision_at_10": 0.0021, + "zero_hit_rate": 0.9149, + "median_tokens_delivered": 32313, + "median_latency_ms": 17.173, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0429, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0143, + "mean_precision_at_10": 0, + "zero_hit_rate": 0.9143, + "median_tokens_delivered": 29377, + "median_latency_ms": 16.646, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 18652.2, + "peak_rss_mb": 18711.9, + "delta_rss_mb": 59.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.1461, + "full_recall_at_5": 0.0366, + "mean_recall_at_10": 0.2191, + "full_recall_at_10": 0.0366, + "mean_recall_at_20": 0.3053, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1187, + "mean_precision_at_10": 0.0488, + "zero_hit_rate": 0.439, + "median_tokens_delivered": 70828, + "median_latency_ms": 42.9225, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.1461, + "full_recall_at_5": 0.0366, + "mean_recall_at_10": 0.2191, + "full_recall_at_10": 0.0366, + "mean_recall_at_20": 0.3053, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1187, + "mean_precision_at_10": 0.0488, + "zero_hit_rate": 0.439, + "median_tokens_delivered": 70828, + "median_latency_ms": 42.9225, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.1344, + "full_recall_at_5": 0.0213, + "mean_recall_at_10": 0.2085, + "full_recall_at_10": 0.0213, + "mean_recall_at_20": 0.2986, + "full_recall_at_20": 0.0426, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1184, + "mean_precision_at_10": 0.0532, + "zero_hit_rate": 0.4043, + "median_tokens_delivered": 71582, + "median_latency_ms": 43.097, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.1619, + "full_recall_at_5": 0.0571, + "mean_recall_at_10": 0.2333, + "full_recall_at_10": 0.0571, + "mean_recall_at_20": 0.3143, + "full_recall_at_20": 0.1143, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.119, + "mean_precision_at_10": 0.0429, + "zero_hit_rate": 0.4857, + "median_tokens_delivered": 70715, + "median_latency_ms": 42.787, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 18244.8, + "peak_rss_mb": 18330.5, + "delta_rss_mb": 85.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.0866, + "full_recall_at_5": 0.0366, + "mean_recall_at_10": 0.1303, + "full_recall_at_10": 0.0366, + "mean_recall_at_20": 0.203, + "full_recall_at_20": 0.0488, + "mean_recall_at_1000_tokens": 0.1272, + "mean_recall_at_4000_tokens": 0.1762, + "mean_recall_at_16000_tokens": 0.203, + "mean_precision_at_10": 0.0421, + "zero_hit_rate": 0.5976, + "median_tokens_delivered": 2520.5, + "median_latency_ms": 782.588, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.0866, + "full_recall_at_5": 0.0366, + "mean_recall_at_10": 0.1303, + "full_recall_at_10": 0.0366, + "mean_recall_at_20": 0.203, + "full_recall_at_20": 0.0488, + "mean_recall_at_1000_tokens": 0.1272, + "mean_recall_at_4000_tokens": 0.1762, + "mean_recall_at_16000_tokens": 0.203, + "mean_precision_at_10": 0.0421, + "zero_hit_rate": 0.5976, + "median_tokens_delivered": 2520.5, + "median_latency_ms": 782.588, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.0801, + "full_recall_at_5": 0.0426, + "mean_recall_at_10": 0.1245, + "full_recall_at_10": 0.0426, + "mean_recall_at_20": 0.1769, + "full_recall_at_20": 0.0426, + "mean_recall_at_1000_tokens": 0.1085, + "mean_recall_at_4000_tokens": 0.1621, + "mean_recall_at_16000_tokens": 0.1769, + "mean_precision_at_10": 0.0426, + "zero_hit_rate": 0.617, + "median_tokens_delivered": 3780, + "median_latency_ms": 798.964, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.0952, + "full_recall_at_5": 0.0286, + "mean_recall_at_10": 0.1381, + "full_recall_at_10": 0.0286, + "mean_recall_at_20": 0.2381, + "full_recall_at_20": 0.0571, + "mean_recall_at_1000_tokens": 0.1524, + "mean_recall_at_4000_tokens": 0.1952, + "mean_recall_at_16000_tokens": 0.2381, + "mean_precision_at_10": 0.0415, + "zero_hit_rate": 0.5714, + "median_tokens_delivered": 1010, + "median_latency_ms": 778.724, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 18327.2, + "peak_rss_mb": 20026.5, + "delta_rss_mb": 1699.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 2271006720 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55380, + "latency_ms": 27.351, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14735, + "latency_ms": 18.73, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33582, + "latency_ms": 16.468, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40619, + "latency_ms": 16.139, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34349, + "latency_ms": 14.996, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49787, + "latency_ms": 18.169, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19564, + "latency_ms": 16.514, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22900, + "latency_ms": 16.848, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [ + "test/types/request.test-d.ts" + ], + "missed": [ + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 29021, + "latency_ms": 17.374, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42213, + "latency_ms": 18.036, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25050, + "latency_ms": 16.635, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47319, + "latency_ms": 17.28, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 25876, + "latency_ms": 17.746, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 52479, + "latency_ms": 17.438, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [], + "missed": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14502, + "latency_ms": 16.891, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js" + ], + "missed": [ + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 15, + "tokens_delivered": 18095, + "latency_ms": 15.912, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29377, + "latency_ms": 15.451, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31292, + "latency_ms": 15.791, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45307, + "latency_ms": 14.482, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18048, + "latency_ms": 15.916, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28081, + "latency_ms": 15.885, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29389, + "latency_ms": 16.336, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49530, + "latency_ms": 14.557, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28931, + "latency_ms": 16.068, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23861, + "latency_ms": 16.355, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30703, + "latency_ms": 15.704, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28832, + "latency_ms": 16.445, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19751, + "latency_ms": 15.232, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 38242, + "latency_ms": 16.032, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33259, + "latency_ms": 17.99, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29737, + "latency_ms": 18.9, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30714, + "latency_ms": 17.68, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26147, + "latency_ms": 17.934, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23070, + "latency_ms": 19.242, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25310, + "latency_ms": 16.943, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/find-route.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33788, + "latency_ms": 17.156, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31982, + "latency_ms": 16.491, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10713, + "latency_ms": 17.145, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18276, + "latency_ms": 16.048, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30719, + "latency_ms": 17.507, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34315, + "latency_ms": 16.487, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42597, + "latency_ms": 16.892, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32783, + "latency_ms": 16.067, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/404s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14947, + "latency_ms": 16.423, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22479, + "latency_ms": 16.33, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30757, + "latency_ms": 17.173, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30239, + "latency_ms": 18.083, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18963, + "latency_ms": 17.246, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "test/hooks.test.js" + ], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 56605, + "latency_ms": 16.533, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29992, + "latency_ms": 17.034, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28153, + "latency_ms": 16.334, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30262, + "latency_ms": 15.855, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45885, + "latency_ms": 17.964, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23215, + "latency_ms": 18.043, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36564, + "latency_ms": 16.678, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46897, + "latency_ms": 16.445, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32459, + "latency_ms": 16.077, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22393, + "latency_ms": 17.143, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28957, + "latency_ms": 16.646, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51533, + "latency_ms": 15.985, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30177, + "latency_ms": 17.16, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42868, + "latency_ms": 18.27, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34695, + "latency_ms": 18.405, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40034, + "latency_ms": 18.509, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46434, + "latency_ms": 17.333, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36331, + "latency_ms": 18.73, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30307, + "latency_ms": 18.802, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40300, + "latency_ms": 19.512, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/plugin-utils.js" + ], + "missed": [ + "lib/errors.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 32313, + "latency_ms": 17.599, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/decorate.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 28910, + "latency_ms": 18.204, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43992, + "latency_ms": 18.068, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23870, + "latency_ms": 17.306, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42687, + "latency_ms": 17.053, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26329, + "latency_ms": 16.273, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28075, + "latency_ms": 17.788, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46278, + "latency_ms": 18.174, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38893, + "latency_ms": 17.752, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22541, + "latency_ms": 17.948, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45100, + "latency_ms": 17.559, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [], + "missed": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17721, + "latency_ms": 17.593, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26451, + "latency_ms": 16.908, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24066, + "latency_ms": 17.304, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70557, + "latency_ms": 43.746, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82986, + "latency_ms": 45.218, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "fastify.js", + "types/instance.d.ts" + ], + "missed": [ + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 70557, + "latency_ms": 37.781, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts" + ], + "missed": [ + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 70629, + "latency_ms": 38.243, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 87404, + "latency_ms": 41.714, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [ + "fastify.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 63466, + "latency_ms": 38.331, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70776, + "latency_ms": 37.408, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70662, + "latency_ms": 37.014, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69669, + "latency_ms": 38.233, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/request-error.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 66533, + "latency_ms": 39.004, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "fastify.js" + ], + "missed": [ + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 65423, + "latency_ms": 39.192, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 71928, + "latency_ms": 41.626, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 74650, + "latency_ms": 45.197, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70662, + "latency_ms": 46.521, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [], + "missed": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70557, + "latency_ms": 48.646, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "lib/route.js" + ], + "missed": [ + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 74728, + "latency_ms": 44.83, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69389, + "latency_ms": 42.787, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/http2/plain.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 71207, + "latency_ms": 50.262, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82986, + "latency_ms": 45.599, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64300, + "latency_ms": 42.493, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70546, + "latency_ms": 40.47, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 89027, + "latency_ms": 42.447, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [ + "fastify.js", + "lib/server.js" + ], + "missed": [ + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 82521, + "latency_ms": 42.616, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 80366, + "latency_ms": 46.84, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 74508, + "latency_ms": 45.01, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 64846, + "latency_ms": 41.802, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74429, + "latency_ms": 40.149, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/errors.js", + "test/web-api.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 84523, + "latency_ms": 42.427, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71253, + "latency_ms": 37.978, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 74448, + "latency_ms": 41.185, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 82525, + "latency_ms": 45.882, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62953, + "latency_ms": 39.278, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63778, + "latency_ms": 40.874, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 60591, + "latency_ms": 42.274, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63424, + "latency_ms": 43.294, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/find-route.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 63892, + "latency_ms": 41.855, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80545, + "latency_ms": 46.557, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 63909, + "latency_ms": 50.038, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68086, + "latency_ms": 45.485, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 84613, + "latency_ms": 48.369, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/reply.js", + "lib/request.js" + ], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 5, + "tokens_delivered": 63724, + "latency_ms": 42.953, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 70641, + "latency_ms": 41.047, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 61937, + "latency_ms": 43.203, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/404s.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 72316, + "latency_ms": 46.739, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "lib/route.js" + ], + "missed": [ + "test/hooks.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 63201, + "latency_ms": 40.483, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82525, + "latency_ms": 44.689, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64345, + "latency_ms": 44.356, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54068, + "latency_ms": 42.477, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 62953, + "latency_ms": 39.098, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/internals/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 69439, + "latency_ms": 44.183, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 74456, + "latency_ms": 41.446, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 68086, + "latency_ms": 39.078, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84659, + "latency_ms": 40.098, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70880, + "latency_ms": 41.819, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 70638, + "latency_ms": 39.688, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74650, + "latency_ms": 38.364, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63886, + "latency_ms": 42.557, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/fastify-instance.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 86006, + "latency_ms": 42.892, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70662, + "latency_ms": 40.591, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 84631, + "latency_ms": 40.786, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 89074, + "latency_ms": 43.373, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64345, + "latency_ms": 43.751, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 72779, + "latency_ms": 43.235, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts" + ], + "missed": [ + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 65418, + "latency_ms": 43.272, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74997, + "latency_ms": 38.616, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63778, + "latency_ms": 42.591, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 74705, + "latency_ms": 43.097, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 72798, + "latency_ms": 47.746, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js" + ], + "missed": [ + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 71582, + "latency_ms": 46.84, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/reply.js", + "lib/request.js" + ], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 5, + "tokens_delivered": 63025, + "latency_ms": 45.931, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 72300, + "latency_ms": 51.863, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 74694, + "latency_ms": 41.389, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64846, + "latency_ms": 45.625, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74728, + "latency_ms": 47.648, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 74702, + "latency_ms": 53.163, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 72267, + "latency_ms": 58.366, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 72267, + "latency_ms": 52.76, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 84982, + "latency_ms": 55.437, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 70715, + "latency_ms": 53.343, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "test/types/instance.test-d.ts" + ], + "missed": [ + "build/build-validation.js", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 65361, + "latency_ms": 52.669, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84594, + "latency_ms": 57.338, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 84607, + "latency_ms": 50.43, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [ + "lib/hooks.js" + ], + "missed": [ + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 5517, + "latency_ms": 898.367, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2363, + "latency_ms": 952.225, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 2793, + "latency_ms": 995.983, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8279, + "latency_ms": 1108.697, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 309, + "latency_ms": 903.843, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [ + "fastify.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2392, + "latency_ms": 931.681, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3554, + "latency_ms": 1051.485, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [ + "test/content-parser.test.js" + ], + "missed": [ + "lib/content-type-parser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 2284, + "latency_ms": 935.836, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1397, + "latency_ms": 886.349, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3780, + "latency_ms": 614.125, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [ + "fastify.d.ts" + ], + "missed": [ + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 3762, + "latency_ms": 540.917, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [ + "test/content-parser.test.js" + ], + "missed": [ + "lib/content-type-parser.js", + "lib/errors.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 449, + "latency_ms": 493.332, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "test/trust-proxy.test.js" + ], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 768, + "latency_ms": 451.504, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2530, + "latency_ms": 420.631, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [ + "test/internals/reply.test.js" + ], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 18, + "tokens_delivered": 2671, + "latency_ms": 429.243, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5771, + "latency_ms": 501.247, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 118, + "latency_ms": 525.315, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 773, + "latency_ms": 514.163, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5858, + "latency_ms": 493.753, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [ + "test/route-prefix.test.js" + ], + "missed": [ + "lib/plugin-override.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 440, + "latency_ms": 630.232, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 807, + "latency_ms": 657.578, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6763, + "latency_ms": 591.057, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1341, + "latency_ms": 647.892, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3808, + "latency_ms": 682.506, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 536, + "latency_ms": 745.351, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 434, + "latency_ms": 765.035, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20, + "latency_ms": 779.101, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 119, + "latency_ms": 632.358, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1010, + "latency_ms": 729.927, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2925, + "latency_ms": 702.437, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 19, + "tokens_delivered": 4770, + "latency_ms": 673.502, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4661, + "latency_ms": 905.954, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3418, + "latency_ms": 1398.737, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4078, + "latency_ms": 1434.455, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6610, + "latency_ms": 882.587, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/find-route.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 1651, + "latency_ms": 815.788, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 317, + "latency_ms": 718.902, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 56, + "latency_ms": 781.546, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6090, + "latency_ms": 743.061, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 5784, + "latency_ms": 798.964, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js" + ], + "missed": [ + "lib/reply.js", + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 16, + "tokens_delivered": 3668, + "latency_ms": 748.703, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 837, + "latency_ms": 769.752, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3933, + "latency_ms": 708.625, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [ + "fastify.js", + "test/404s.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1261, + "latency_ms": 765.554, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "test/hooks.test.js" + ], + "missed": [ + "lib/reply.js", + "lib/route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 6, + "tokens_delivered": 452, + "latency_ms": 772.7, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5750, + "latency_ms": 734.457, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1974, + "latency_ms": 823.335, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6176, + "latency_ms": 862.041, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "test/hooks.test.js" + ], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 2553, + "latency_ms": 877.138, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 16, + "tokens_delivered": 5911, + "latency_ms": 809.35, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3975, + "latency_ms": 827.832, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "test/reply-trailers.test.js" + ], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 344, + "latency_ms": 778.724, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 355, + "latency_ms": 833.969, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5548, + "latency_ms": 783.63, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "test/close.test.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 14, + "tokens_delivered": 5098, + "latency_ms": 859.529, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2563, + "latency_ms": 824.151, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1921, + "latency_ms": 820.536, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/fastify-instance.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 22, + "latency_ms": 775.064, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [ + "lib/log-controller.js" + ], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1906, + "latency_ms": 891.65, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "test/trust-proxy.test.js" + ], + "missed": [ + "lib/request.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 467, + "latency_ms": 864.043, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "types/instance.d.ts" + ], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 7, + "tokens_delivered": 6936, + "latency_ms": 824.977, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [ + "test/internals/handle-request.test.js" + ], + "missed": [ + "lib/handleRequest.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 19, + "tokens_delivered": 2218, + "latency_ms": 809.769, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 396, + "latency_ms": 872.259, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts" + ], + "missed": [ + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 20, + "tokens_delivered": 4466, + "latency_ms": 887.651, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6088, + "latency_ms": 931.556, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 743, + "latency_ms": 878.521, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [ + "test/custom-http-server.test.js" + ], + "missed": [ + "fastify.js", + "test/close-pipelining.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 639, + "latency_ms": 887.875, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8335, + "latency_ms": 880.332, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/plugin-utils.js", + "test/internals/plugin.test.js" + ], + "missed": [ + "lib/errors.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.5, + "recall_at_10": 0.4, + "precision_at_10": 0.5, + "recall_at_20": 0.4, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.4, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 2, + "tokens_delivered": 106, + "latency_ms": 872.181, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 166, + "latency_ms": 908.455, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7889, + "latency_ms": 840.377, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 790, + "latency_ms": 856.701, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 964, + "latency_ms": 834.087, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 699, + "latency_ms": 859.153, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5382, + "latency_ms": 720.699, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6370, + "latency_ms": 643.616, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [ + "test/server.test.js" + ], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 160, + "latency_ms": 636.878, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7247, + "latency_ms": 575.123, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2511, + "latency_ms": 575.241, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "fastify.d.ts" + ], + "missed": [ + "build/build-validation.js", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 15, + "tokens_delivered": 4791, + "latency_ms": 431.392, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2548, + "latency_ms": 405.71, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "test/close.test.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 5174, + "latency_ms": 499.505, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/CV-hugo.json b/benchmarks/context-retrieval/results/CV-hugo.json new file mode 100644 index 00000000..89f32cd6 --- /dev/null +++ b/benchmarks/context-retrieval/results/CV-hugo.json @@ -0,0 +1,2939 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "hugo", + "head": "7b5199fdeff7d7682e9f979f468d55bc7b67b934" + }, + "tier": "medium", + "tier_code_files": 932, + "corpus_counts": { + "cases": 29, + "path_leak": 13, + "baseline_blind": 16, + "multi_file": 25 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": false, + "best": 0, + "control": 0, + "reason": "a query-blind control scored 0.0% against a leader's 0.0%. Suspect the metric, not the tools." + }, + "needs_raw_payload_check": [ + { + "provider_id": "codevetter-structural-context", + "why": "scored exactly zero — read the raw payload before believing it" + } + ], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "Fix potential content XSS by escaping dangerous URLs in links and images", + "revision": "81a5cdca0788ca39574a17d444c9db29d0b19e27", + "returned": [ + "markup/goldmark/images/images_integration_test.go", + "markup/goldmark/images/transform.go", + "tpl/collections/where_test.go", + "modules/collect_test.go", + "markup/goldmark/goldmark_integration_test.go" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix menu pageRef resolution in multidimensional setups", + "revision": "d98cd4aecf25b9df78d811759ea6135b0c7610f1", + "returned": [ + "common/hugo/version_current.go", + "hugolib/site.go", + "hugolib/hugo_sites.go", + "resources/images/config.go", + ".github/workflows/aiwatchdog.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix some default site redirect woes", + "revision": "66ba63cd5f30ef8807fc524dbc67399376299bb1", + "returned": [ + "common/hugo/version_current.go", + "tpl/tplimpl/templatestore.go", + "tpl/tplimpl/templatestore_integration_test.go", + "hugolib/site.go", + ".github/workflows/aiwatchdog.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix index out of range panic in fileEventsContentPaths", + "revision": "e8577771c3275d16b7d1b31cca5e9fd3ea517416", + "returned": [ + "common/hugo/version_current.go", + "hugolib/site.go", + "hugolib/hugo_sites.go", + "resources/images/config.go", + ".github/workflows/aiwatchdog.yml" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.0172, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0172, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0345, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0172, + "mean_precision_at_10": 0.0034, + "zero_hit_rate": 0.931, + "median_tokens_delivered": 31301, + "median_latency_ms": 17.105, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.0172, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0172, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0345, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0172, + "mean_precision_at_10": 0.0034, + "zero_hit_rate": 0.931, + "median_tokens_delivered": 31301, + "median_latency_ms": 17.105, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 33227, + "median_latency_ms": 17.284, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.0313, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0313, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0625, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0313, + "mean_precision_at_10": 0.0063, + "zero_hit_rate": 0.875, + "median_tokens_delivered": 29186, + "median_latency_ms": 16.602, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 18567.9, + "peak_rss_mb": 18589.7, + "delta_rss_mb": 21.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.071, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1026, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1938, + "full_recall_at_20": 0.069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0345, + "zero_hit_rate": 0.5862, + "median_tokens_delivered": 145957, + "median_latency_ms": 42.597, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.071, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1026, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1938, + "full_recall_at_20": 0.069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0345, + "zero_hit_rate": 0.5862, + "median_tokens_delivered": 145957, + "median_latency_ms": 42.597, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.0879, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.12, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2335, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0462, + "zero_hit_rate": 0.4615, + "median_tokens_delivered": 141901, + "median_latency_ms": 44.349, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.0573, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0885, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1615, + "full_recall_at_20": 0.0625, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.025, + "zero_hit_rate": 0.6875, + "median_tokens_delivered": 157334.5, + "median_latency_ms": 41.532, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 18530.6, + "peak_rss_mb": 18530.6, + "delta_rss_mb": 0, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 2999.894, + "payload_kind": [ + "excerpts" + ], + "unavailable": 29 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 2999.894, + "payload_kind": [ + "excerpts" + ], + "unavailable": 29 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 3146.603, + "payload_kind": [ + "excerpts" + ], + "unavailable": 13 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 2967.4245, + "payload_kind": [ + "excerpts" + ], + "unavailable": 16 + } + }, + "outcomes": { + "unavailable-at-query-time": 29 + }, + "resources": { + "baseline_rss_mb": 18510.7, + "peak_rss_mb": 22938.6, + "delta_rss_mb": 4427.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 3606482944 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: a query-blind control scored 0.0% against a leader's 0.0%. Suspect the metric, not the tools.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23450, + "latency_ms": 24.162, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43237, + "latency_ms": 16.735, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [], + "missed": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15701, + "latency_ms": 17.149, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [], + "missed": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23223, + "latency_ms": 17.064, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [], + "missed": [ + "commands/server.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31301, + "latency_ms": 18.424, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33227, + "latency_ms": 17.33, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [ + "markup/goldmark/goldmark_integration_test.go" + ], + "missed": [ + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 31381, + "latency_ms": 16.027, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26363, + "latency_ms": 17.175, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32262, + "latency_ms": 17.162, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28830, + "latency_ms": 18.681, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18869, + "latency_ms": 17.008, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [ + "hugolib/embedded_templates_test.go" + ], + "missed": [ + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 23902, + "latency_ms": 17.158, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27186, + "latency_ms": 19.22, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33298, + "latency_ms": 17.117, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [], + "missed": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38765, + "latency_ms": 17.105, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23248, + "latency_ms": 16.455, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32875, + "latency_ms": 17.486, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33433, + "latency_ms": 15.308, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39664, + "latency_ms": 15.588, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24224, + "latency_ms": 14.931, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22664, + "latency_ms": 15.278, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27862, + "latency_ms": 16.749, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44378, + "latency_ms": 16.135, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84924, + "latency_ms": 17.284, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43205, + "latency_ms": 18.449, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43171, + "latency_ms": 16.741, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [], + "missed": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29542, + "latency_ms": 16.316, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34959, + "latency_ms": 16.915, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30727, + "latency_ms": 17.196, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 135284, + "latency_ms": 58.194, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 141901, + "latency_ms": 36.684, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [ + "hugolib/cascade_test.go" + ], + "missed": [ + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 147188, + "latency_ms": 44.349, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [], + "missed": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 167064, + "latency_ms": 42.683, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [], + "missed": [ + "commands/server.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 158300, + "latency_ms": 44.993, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/site.go" + ], + "missed": [ + "hugolib/menu_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 160960, + "latency_ms": 42.672, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [], + "missed": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 156369, + "latency_ms": 39.92, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123900, + "latency_ms": 71.752, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145957, + "latency_ms": 35.93, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 167158, + "latency_ms": 41.587, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 168902, + "latency_ms": 40.563, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 143955, + "latency_ms": 39.819, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [ + "hugolib/integrationtest_builder.go" + ], + "missed": [ + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 148660, + "latency_ms": 39.286, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/sitesmatrix/dimensions.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.2857, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 167157, + "latency_ms": 37.125, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "config/allconfig/allconfig.go", + "hugolib/site.go" + ], + "missed": [ + ".prettierignore", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 140039, + "latency_ms": 39.71, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 158300, + "latency_ms": 38.192, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145753, + "latency_ms": 42.508, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 133497, + "latency_ms": 59.344, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 141218, + "latency_ms": 43.795, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [ + "hugolib/rebuild_test.go" + ], + "missed": [ + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 167064, + "latency_ms": 41.477, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145952, + "latency_ms": 39.437, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159604, + "latency_ms": 46.945, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [ + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 133592, + "latency_ms": 46.978, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [ + "hugolib/rebuild_test.go" + ], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 135377, + "latency_ms": 66.436, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 139252, + "latency_ms": 47.931, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 133591, + "latency_ms": 48.137, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [ + "tpl/css/build_integration_test.go" + ], + "missed": [ + "resources/transform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 158549, + "latency_ms": 40.084, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 163147, + "latency_ms": 44.745, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 138542, + "latency_ms": 42.597, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1868.343, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1775.047, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [], + "missed": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2113.965, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [], + "missed": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2266.95, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [], + "missed": [ + "commands/server.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2631.277, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2665.64, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [], + "missed": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2536.694, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2558.591, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 4000.256, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2615.969, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2934.955, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2999.894, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 3466.809, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 3253.09, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [], + "missed": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 4025.853, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 4128.861, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 4308.883, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2635.995, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1945.848, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2275.198, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 3131.343, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 3605.329, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 3259.141, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 3751.217, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 3146.603, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 4611.31, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [], + "missed": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5237.726, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 3906.745, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2848.683, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/M-fastify-lex.json b/benchmarks/context-retrieval/results/M-fastify-lex.json new file mode 100644 index 00000000..d7f1db4e --- /dev/null +++ b/benchmarks/context-retrieval/results/M-fastify-lex.json @@ -0,0 +1,14576 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "fastify", + "head": "83e69762aff047940b376fc2a266d143499ef727" + }, + "tier": "medium", + "tier_code_files": 300, + "corpus_counts": { + "cases": 82, + "path_leak": 47, + "baseline_blind": 35, + "multi_file": 66 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.1421, + "control": 0, + "reason": null + }, + "needs_raw_payload_check": [ + { + "provider_id": "zoekt", + "why": "scored exactly zero — read the raw payload before believing it" + } + ], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "replace AssertionError with FST_ERR_PLUGIN_DEPENDENCY_NOT_REGISTERED in checkDependencies (#6774)", + "revision": "27f4d6222dd945238e96c1fdaa632d5dc3c37b2c", + "returned": [ + ".github/workflows/coverage-nix.yml", + "examples/route-prefix.js", + "test/internals/hooks.test.js", + "lib/content-type.js", + "test/router-options.test.js" + ] + }, + { + "provider_id": "churn-ranked", + "query": "add http method override warning (#6879)", + "revision": "b93d611c7cefc2cd4a0b6f05cc864f2b6c397a6d", + "returned": [ + "package.json", + "fastify.js", + ".github/workflows/ci.yml", + "lib/reply.js", + ".github/workflows/links-check.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "use JSON.stringify in onBadUrl for proper escaping (#6420)", + "revision": "970c575832521fff01cc018d928d84454811173b", + "returned": [ + "package.json", + "fastify.js", + ".github/workflows/ci.yml", + "lib/reply.js", + "lib/route.js" + ] + }, + { + "provider_id": "churn-ranked", + "query": "run preClose hook exactly once per declaring instance (#6940)", + "revision": "9a1535bcfe2a780cc6f9c6555df86e5072ca9a29", + "returned": [ + "package.json", + "fastify.js", + ".github/workflows/ci.yml", + "lib/reply.js", + "fastify.d.ts" + ] + }, + { + "provider_id": "ripgrep", + "query": "add http method override warning (#6879)", + "revision": "b93d611c7cefc2cd4a0b6f05cc864f2b6c397a6d", + "returned": [ + "fastify.js", + "test/content-parser.test.js", + "test/custom-parser.0.test.js", + "test/hooks.test.js", + "fastify.d.ts" + ] + }, + { + "provider_id": "ripgrep", + "query": "clear `[kState].readyPromise` for garbage collection (#6030)", + "revision": "7868e47194bcfb34cab2e51ea36bf10f764b538e", + "returned": [ + "fastify.js", + "lib/server.js", + "test/hooks.test.js", + "test/plugin.4.test.js", + "types/hooks.d.ts" + ] + }, + { + "provider_id": "ripgrep", + "query": "Align routerOptions defaultRoute types with runtime (#6392)", + "revision": "4c10980bd4a5a4d19e051f52a018779e6afb588b", + "returned": [ + "fastify.d.ts", + "fastify.js", + "build/build-validation.js", + "lib/route.js", + "types/instance.d.ts" + ] + }, + { + "provider_id": "ast-grep", + "query": "add http method override warning (#6879)", + "revision": "b93d611c7cefc2cd4a0b6f05cc864f2b6c397a6d", + "returned": [ + "types/server-factory.d.ts", + "fastify.d.ts", + "lib/server.js", + "types/utils.d.ts", + "fastify.js" + ] + }, + { + "provider_id": "ast-grep", + "query": "clear trailer state when removing all trailers (#6845)", + "revision": "043baf988048c08ee87190193089a3b714f163fe", + "returned": [ + "test/internals/hook-runner.test.js", + "lib/reply.js" + ] + }, + { + "provider_id": "ast-grep", + "query": "Align routerOptions defaultRoute types with runtime (#6392)", + "revision": "4c10980bd4a5a4d19e051f52a018779e6afb588b", + "returned": [ + "fastify.d.ts", + "lib/route.js", + "test/schema-special-usage.test.js", + "test/esm/plugin.mjs", + "test/esm/other.mjs" + ] + }, + { + "provider_id": "filename-match", + "query": "add http method override warning (#6879)", + "revision": "b93d611c7cefc2cd4a0b6f05cc864f2b6c397a6d", + "returned": [ + "test/http2/unknown-http-method.test.js", + "lib/plugin-override.js", + "test/custom-http-server.test.js", + "test/http-methods/copy.test.js", + "test/http-methods/custom-http-methods.test.js" + ] + }, + { + "provider_id": "filename-match", + "query": "clear trailer state when removing all trailers (#6845)", + "revision": "043baf988048c08ee87190193089a3b714f163fe", + "returned": [ + "test/reply-trailers.test.js" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0024, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0329, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0091, + "mean_precision_at_10": 0.0012, + "zero_hit_rate": 0.9146, + "median_tokens_delivered": 30284.5, + "median_latency_ms": 11.068, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0024, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0329, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0091, + "mean_precision_at_10": 0.0012, + "zero_hit_rate": 0.9146, + "median_tokens_delivered": 30284.5, + "median_latency_ms": 11.068, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0043, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0255, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0053, + "mean_precision_at_10": 0.0021, + "zero_hit_rate": 0.9149, + "median_tokens_delivered": 32313, + "median_latency_ms": 11.054, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0429, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0143, + "mean_precision_at_10": 0, + "zero_hit_rate": 0.9143, + "median_tokens_delivered": 29377, + "median_latency_ms": 11.109, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 20202.5, + "peak_rss_mb": 20221.6, + "delta_rss_mb": 19.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.1461, + "full_recall_at_5": 0.0366, + "mean_recall_at_10": 0.2191, + "full_recall_at_10": 0.0366, + "mean_recall_at_20": 0.3053, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1187, + "mean_precision_at_10": 0.0488, + "zero_hit_rate": 0.439, + "median_tokens_delivered": 70828, + "median_latency_ms": 25.021, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.1461, + "full_recall_at_5": 0.0366, + "mean_recall_at_10": 0.2191, + "full_recall_at_10": 0.0366, + "mean_recall_at_20": 0.3053, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1187, + "mean_precision_at_10": 0.0488, + "zero_hit_rate": 0.439, + "median_tokens_delivered": 70828, + "median_latency_ms": 25.021, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.1344, + "full_recall_at_5": 0.0213, + "mean_recall_at_10": 0.2085, + "full_recall_at_10": 0.0213, + "mean_recall_at_20": 0.2986, + "full_recall_at_20": 0.0426, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1184, + "mean_precision_at_10": 0.0532, + "zero_hit_rate": 0.4043, + "median_tokens_delivered": 71582, + "median_latency_ms": 24.852, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.1619, + "full_recall_at_5": 0.0571, + "mean_recall_at_10": 0.2333, + "full_recall_at_10": 0.0571, + "mean_recall_at_20": 0.3143, + "full_recall_at_20": 0.1143, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.119, + "mean_precision_at_10": 0.0429, + "zero_hit_rate": 0.4857, + "median_tokens_delivered": 70715, + "median_latency_ms": 25.112, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 20142.1, + "peak_rss_mb": 20168.7, + "delta_rss_mb": 26.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "ripgrep", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.3955, + "full_recall_at_5": 0.2195, + "mean_recall_at_10": 0.5118, + "full_recall_at_10": 0.3171, + "mean_recall_at_20": 0.6337, + "full_recall_at_20": 0.4512, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0396, + "mean_recall_at_16000_tokens": 0.2937, + "mean_precision_at_10": 0.1098, + "zero_hit_rate": 0.1951, + "median_tokens_delivered": 103773, + "median_latency_ms": 136.796, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.3955, + "full_recall_at_5": 0.2195, + "mean_recall_at_10": 0.5118, + "full_recall_at_10": 0.3171, + "mean_recall_at_20": 0.6337, + "full_recall_at_20": 0.4512, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0396, + "mean_recall_at_16000_tokens": 0.2937, + "mean_precision_at_10": 0.1098, + "zero_hit_rate": 0.1951, + "median_tokens_delivered": 103773, + "median_latency_ms": 136.796, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.4262, + "full_recall_at_5": 0.234, + "mean_recall_at_10": 0.4908, + "full_recall_at_10": 0.2979, + "mean_recall_at_20": 0.6255, + "full_recall_at_20": 0.4468, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0372, + "mean_recall_at_16000_tokens": 0.2954, + "mean_precision_at_10": 0.1213, + "zero_hit_rate": 0.2128, + "median_tokens_delivered": 103601, + "median_latency_ms": 137.574, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.3543, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.54, + "full_recall_at_10": 0.3429, + "mean_recall_at_20": 0.6448, + "full_recall_at_20": 0.4571, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0429, + "mean_recall_at_16000_tokens": 0.2914, + "mean_precision_at_10": 0.0943, + "zero_hit_rate": 0.1714, + "median_tokens_delivered": 103945, + "median_latency_ms": 131.508, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 20169, + "peak_rss_mb": 20331.2, + "delta_rss_mb": 162.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "zoekt", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 46.6065, + "payload_kind": [ + "whole-files" + ], + "unavailable": 82 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 46.6065, + "payload_kind": [ + "whole-files" + ], + "unavailable": 82 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 46.801, + "payload_kind": [ + "whole-files" + ], + "unavailable": 47 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 46.04, + "payload_kind": [ + "whole-files" + ], + "unavailable": 35 + } + }, + "outcomes": { + "did-not-index": 82 + }, + "resources": { + "baseline_rss_mb": 19914, + "peak_rss_mb": 19986.4, + "delta_rss_mb": 72.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "ast-grep", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.1165, + "full_recall_at_5": 0.0366, + "mean_recall_at_10": 0.1748, + "full_recall_at_10": 0.061, + "mean_recall_at_20": 0.2697, + "full_recall_at_20": 0.0976, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0051, + "mean_recall_at_16000_tokens": 0.122, + "mean_precision_at_10": 0.0565, + "zero_hit_rate": 0.5122, + "median_tokens_delivered": 42986.5, + "median_latency_ms": 251.4745, + "payload_kind": [ + "whole-files" + ], + "unavailable": 3 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.1165, + "full_recall_at_5": 0.0366, + "mean_recall_at_10": 0.1748, + "full_recall_at_10": 0.061, + "mean_recall_at_20": 0.2697, + "full_recall_at_20": 0.0976, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0051, + "mean_recall_at_16000_tokens": 0.122, + "mean_precision_at_10": 0.0565, + "zero_hit_rate": 0.5122, + "median_tokens_delivered": 42986.5, + "median_latency_ms": 251.4745, + "payload_kind": [ + "whole-files" + ], + "unavailable": 3 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.1245, + "full_recall_at_5": 0.0213, + "mean_recall_at_10": 0.166, + "full_recall_at_10": 0.0213, + "mean_recall_at_20": 0.289, + "full_recall_at_20": 0.0638, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0089, + "mean_recall_at_16000_tokens": 0.1234, + "mean_precision_at_10": 0.0556, + "zero_hit_rate": 0.4255, + "median_tokens_delivered": 45132, + "median_latency_ms": 268.482, + "payload_kind": [ + "whole-files" + ], + "unavailable": 1 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.1057, + "full_recall_at_5": 0.0571, + "mean_recall_at_10": 0.1867, + "full_recall_at_10": 0.1143, + "mean_recall_at_20": 0.2438, + "full_recall_at_20": 0.1429, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.12, + "mean_precision_at_10": 0.0579, + "zero_hit_rate": 0.6286, + "median_tokens_delivered": 36728, + "median_latency_ms": 228.141, + "payload_kind": [ + "whole-files" + ], + "unavailable": 2 + } + }, + "outcomes": { + "answered": 77, + "indexed-but-returned-nothing": 5 + }, + "resources": { + "baseline_rss_mb": 19925, + "peak_rss_mb": 21392, + "delta_rss_mb": 1467.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "filename-match", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.2451, + "full_recall_at_5": 0.1098, + "mean_recall_at_10": 0.2805, + "full_recall_at_10": 0.1585, + "mean_recall_at_20": 0.3173, + "full_recall_at_20": 0.1829, + "mean_recall_at_1000_tokens": 0.0122, + "mean_recall_at_4000_tokens": 0.1421, + "mean_recall_at_16000_tokens": 0.2774, + "mean_precision_at_10": 0.1125, + "zero_hit_rate": 0.4878, + "median_tokens_delivered": 17808, + "median_latency_ms": 10.8545, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.2451, + "full_recall_at_5": 0.1098, + "mean_recall_at_10": 0.2805, + "full_recall_at_10": 0.1585, + "mean_recall_at_20": 0.3173, + "full_recall_at_20": 0.1829, + "mean_recall_at_1000_tokens": 0.0122, + "mean_recall_at_4000_tokens": 0.1421, + "mean_recall_at_16000_tokens": 0.2774, + "mean_precision_at_10": 0.1125, + "zero_hit_rate": 0.4878, + "median_tokens_delivered": 17808, + "median_latency_ms": 10.8545, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.4277, + "full_recall_at_5": 0.1915, + "mean_recall_at_10": 0.4894, + "full_recall_at_10": 0.2766, + "mean_recall_at_20": 0.5535, + "full_recall_at_20": 0.3191, + "mean_recall_at_1000_tokens": 0.0213, + "mean_recall_at_4000_tokens": 0.2479, + "mean_recall_at_16000_tokens": 0.484, + "mean_precision_at_10": 0.1962, + "zero_hit_rate": 0.1064, + "median_tokens_delivered": 29594, + "median_latency_ms": 10.933, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 5188, + "median_latency_ms": 10.599, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 70, + "indexed-but-returned-nothing": 12 + }, + "resources": { + "baseline_rss_mb": 20574.3, + "peak_rss_mb": 20639.1, + "delta_rss_mb": 64.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55380, + "latency_ms": 24.007, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14735, + "latency_ms": 13.874, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33582, + "latency_ms": 11.402, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40619, + "latency_ms": 11.759, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34349, + "latency_ms": 11.918, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49787, + "latency_ms": 12.575, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19564, + "latency_ms": 11.278, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22900, + "latency_ms": 11.638, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [ + "test/types/request.test-d.ts" + ], + "missed": [ + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 29021, + "latency_ms": 15.446, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42213, + "latency_ms": 12.077, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25050, + "latency_ms": 10.92, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47319, + "latency_ms": 11.899, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 25876, + "latency_ms": 11.321, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 52479, + "latency_ms": 10.78, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [], + "missed": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14502, + "latency_ms": 11.062, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js" + ], + "missed": [ + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 15, + "tokens_delivered": 18095, + "latency_ms": 10.48, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29377, + "latency_ms": 11.382, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31292, + "latency_ms": 10.773, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45307, + "latency_ms": 10.124, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18048, + "latency_ms": 10.927, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28081, + "latency_ms": 10.275, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29389, + "latency_ms": 11.054, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49530, + "latency_ms": 10.717, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28931, + "latency_ms": 11.918, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23861, + "latency_ms": 11.289, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30703, + "latency_ms": 11.231, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28832, + "latency_ms": 11.109, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19751, + "latency_ms": 10.448, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 38242, + "latency_ms": 10.946, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33259, + "latency_ms": 10.769, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29737, + "latency_ms": 10.502, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30714, + "latency_ms": 10.734, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26147, + "latency_ms": 11.491, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23070, + "latency_ms": 12.309, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25310, + "latency_ms": 11.544, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/find-route.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33788, + "latency_ms": 11.242, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31982, + "latency_ms": 10.708, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10713, + "latency_ms": 11.43, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18276, + "latency_ms": 10.206, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30719, + "latency_ms": 10.167, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34315, + "latency_ms": 10.359, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42597, + "latency_ms": 11.183, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32783, + "latency_ms": 10.353, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/404s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14947, + "latency_ms": 11.797, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22479, + "latency_ms": 11.344, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30757, + "latency_ms": 10.228, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30239, + "latency_ms": 11.034, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18963, + "latency_ms": 11.133, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "test/hooks.test.js" + ], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 56605, + "latency_ms": 10.493, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29992, + "latency_ms": 10.862, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28153, + "latency_ms": 10.462, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30262, + "latency_ms": 10.576, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45885, + "latency_ms": 11.074, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23215, + "latency_ms": 10.798, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36564, + "latency_ms": 10.474, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46897, + "latency_ms": 11.234, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32459, + "latency_ms": 11.443, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22393, + "latency_ms": 11.655, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28957, + "latency_ms": 12.01, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51533, + "latency_ms": 11.214, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30177, + "latency_ms": 10.594, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42868, + "latency_ms": 10.454, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34695, + "latency_ms": 11.417, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40034, + "latency_ms": 10.945, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46434, + "latency_ms": 11.166, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36331, + "latency_ms": 11.224, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30307, + "latency_ms": 10.312, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40300, + "latency_ms": 10.57, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/plugin-utils.js" + ], + "missed": [ + "lib/errors.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 32313, + "latency_ms": 10.119, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/decorate.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 28910, + "latency_ms": 10.32, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43992, + "latency_ms": 10.39, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23870, + "latency_ms": 11.109, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42687, + "latency_ms": 11.048, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26329, + "latency_ms": 11.125, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28075, + "latency_ms": 11.575, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46278, + "latency_ms": 11.811, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38893, + "latency_ms": 11.357, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22541, + "latency_ms": 11.089, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45100, + "latency_ms": 10.524, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [], + "missed": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17721, + "latency_ms": 10.466, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26451, + "latency_ms": 10.587, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24066, + "latency_ms": 10.793, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70557, + "latency_ms": 26.222, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82986, + "latency_ms": 31.294, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "fastify.js", + "types/instance.d.ts" + ], + "missed": [ + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 70557, + "latency_ms": 23.419, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts" + ], + "missed": [ + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 70629, + "latency_ms": 24.805, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 87404, + "latency_ms": 27.383, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [ + "fastify.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 63466, + "latency_ms": 24.677, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70776, + "latency_ms": 24.744, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70662, + "latency_ms": 24.835, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69669, + "latency_ms": 26.176, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/request-error.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 66533, + "latency_ms": 26.687, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "fastify.js" + ], + "missed": [ + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 65423, + "latency_ms": 24.344, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 71928, + "latency_ms": 24.625, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 74650, + "latency_ms": 23.603, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70662, + "latency_ms": 24.643, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [], + "missed": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70557, + "latency_ms": 25.114, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "lib/route.js" + ], + "missed": [ + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 74728, + "latency_ms": 24.029, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69389, + "latency_ms": 23.939, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/http2/plain.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 71207, + "latency_ms": 24.205, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82986, + "latency_ms": 25.98, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64300, + "latency_ms": 24.532, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70546, + "latency_ms": 24.952, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 89027, + "latency_ms": 26.301, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [ + "fastify.js", + "lib/server.js" + ], + "missed": [ + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 82521, + "latency_ms": 25.795, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 80366, + "latency_ms": 26.696, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 74508, + "latency_ms": 23.794, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 64846, + "latency_ms": 24.997, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74429, + "latency_ms": 24.801, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/errors.js", + "test/web-api.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 84523, + "latency_ms": 27.128, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71253, + "latency_ms": 24.411, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 74448, + "latency_ms": 23.866, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 82525, + "latency_ms": 26.154, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62953, + "latency_ms": 24.7, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63778, + "latency_ms": 25.429, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 60591, + "latency_ms": 24.852, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63424, + "latency_ms": 25.312, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/find-route.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 63892, + "latency_ms": 24.675, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80545, + "latency_ms": 25.888, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 63909, + "latency_ms": 23.393, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68086, + "latency_ms": 24.019, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 84613, + "latency_ms": 24.788, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/reply.js", + "lib/request.js" + ], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 5, + "tokens_delivered": 63724, + "latency_ms": 24.696, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 70641, + "latency_ms": 24.015, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 61937, + "latency_ms": 25.888, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/404s.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 72316, + "latency_ms": 25.561, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "lib/route.js" + ], + "missed": [ + "test/hooks.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 63201, + "latency_ms": 23.641, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82525, + "latency_ms": 25.821, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64345, + "latency_ms": 25.466, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54068, + "latency_ms": 24.718, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 62953, + "latency_ms": 24.246, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/internals/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 69439, + "latency_ms": 26.231, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 74456, + "latency_ms": 23.604, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 68086, + "latency_ms": 23.701, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84659, + "latency_ms": 26.551, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70880, + "latency_ms": 27.494, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 70638, + "latency_ms": 24.418, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74650, + "latency_ms": 23.574, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63886, + "latency_ms": 25.382, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/fastify-instance.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 86006, + "latency_ms": 26.74, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70662, + "latency_ms": 24.968, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 84631, + "latency_ms": 25.873, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 89074, + "latency_ms": 27.444, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64345, + "latency_ms": 27.177, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 72779, + "latency_ms": 25.255, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts" + ], + "missed": [ + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 65418, + "latency_ms": 26.622, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74997, + "latency_ms": 24.462, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63778, + "latency_ms": 26.319, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 74705, + "latency_ms": 24.648, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 72798, + "latency_ms": 24.731, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js" + ], + "missed": [ + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 71582, + "latency_ms": 25.045, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/reply.js", + "lib/request.js" + ], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 5, + "tokens_delivered": 63025, + "latency_ms": 25.458, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 72300, + "latency_ms": 25.968, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 74694, + "latency_ms": 23.751, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64846, + "latency_ms": 25.112, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74728, + "latency_ms": 26.248, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 74702, + "latency_ms": 26.315, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 72267, + "latency_ms": 26.514, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 72267, + "latency_ms": 25.13, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 84982, + "latency_ms": 25.11, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 70715, + "latency_ms": 23.849, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "test/types/instance.test-d.ts" + ], + "missed": [ + "build/build-validation.js", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 65361, + "latency_ms": 24.697, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84594, + "latency_ms": 26.335, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 84607, + "latency_ms": 25.584, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [ + "test/reply-error.test.js" + ], + "missed": [ + "lib/hooks.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 122182, + "latency_ms": 150.54, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67691, + "latency_ms": 105.709, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "fastify.js", + "lib/warnings.js" + ], + "missed": [ + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 122295, + "latency_ms": 118.892, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 122718, + "latency_ms": 137.32, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "types/instance.d.ts" + ], + "missed": [ + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 67617, + "latency_ms": 105.083, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [ + "fastify.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 79579, + "latency_ms": 110.038, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 123787, + "latency_ms": 131.508, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 93462, + "latency_ms": 171.812, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 94851, + "latency_ms": 125.022, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [ + "lib/request.js", + "test/request-error.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 68051, + "latency_ms": 116.998, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "fastify.js" + ], + "missed": [ + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 112964, + "latency_ms": 112.873, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 114483, + "latency_ms": 147.313, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 62694, + "latency_ms": 138.428, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 117798, + "latency_ms": 159.714, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [ + "test/internals/reply.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 106900, + "latency_ms": 147.633, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "lib/errors.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 104642, + "latency_ms": 123.916, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46880, + "latency_ms": 112.924, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/http2/plain.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 100963, + "latency_ms": 127.18, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49265, + "latency_ms": 118.875, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/plugin-override.js" + ], + "missed": [ + "test/route-prefix.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 97637, + "latency_ms": 139.611, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68093, + "latency_ms": 126.333, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45900, + "latency_ms": 107.888, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [ + "fastify.js", + "lib/server.js", + "test/http2/closing.test.js" + ], + "missed": [ + "lib/symbols.js" + ], + "recall_at_5": 0.75, + "precision_at_5": 0.6, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 2, + "tokens_delivered": 61669, + "latency_ms": 117.794, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 107111, + "latency_ms": 145.636, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 110425, + "latency_ms": 99.341, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 86161, + "latency_ms": 115.254, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 55573, + "latency_ms": 158.469, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [ + "lib/errors.js", + "lib/reply.js" + ], + "missed": [ + "test/web-api.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 116360, + "latency_ms": 139.041, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "fastify.js", + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 121245, + "latency_ms": 141.741, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 83432, + "latency_ms": 149.306, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 130386, + "latency_ms": 142.683, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 77995, + "latency_ms": 130.298, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [ + "test/helper.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 105097, + "latency_ms": 125.202, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [ + "fastify.js", + "lib/server.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 97321, + "latency_ms": 138.573, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [ + "test/close-pipelining.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 58970, + "latency_ms": 137.813, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js", + "test/find-route.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 100748, + "latency_ms": 123.082, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [ + "test/content-parser.test.js" + ], + "missed": [ + "lib/content-type-parser.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 73995, + "latency_ms": 102.035, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 63677, + "latency_ms": 122.752, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [ + "lib/handle-request.js" + ], + "missed": [ + "lib/error-handler.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 63246, + "latency_ms": 139.013, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 119274, + "latency_ms": 109.479, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/content-type-parser.js", + "lib/reply.js" + ], + "missed": [ + "lib/content-type.js", + "lib/handle-request.js", + "lib/request.js" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 99465, + "latency_ms": 146.016, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 95568, + "latency_ms": 127.641, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [ + "test/hooks-async.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 77375, + "latency_ms": 107.084, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [ + "test/404s.test.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 115276, + "latency_ms": 120.557, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 109415, + "latency_ms": 146.47, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 130157, + "latency_ms": 137.769, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [ + "types/logger.d.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 84879, + "latency_ms": 126.801, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 77052, + "latency_ms": 110.597, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 124179, + "latency_ms": 157.248, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 118496, + "latency_ms": 164.373, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 103601, + "latency_ms": 183.962, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 103945, + "latency_ms": 120.422, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 120633, + "latency_ms": 118.274, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [ + "lib/error-handler.js", + "lib/handle-request.js" + ], + "missed": [ + "lib/error-status.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 36837, + "latency_ms": 137.574, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js", + "test/close.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 86026, + "latency_ms": 130.154, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117282, + "latency_ms": 148.633, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 117223, + "latency_ms": 157.566, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 120178, + "latency_ms": 152.295, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [ + "types/logger.d.ts" + ], + "missed": [ + "lib/log-controller.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 132835, + "latency_ms": 150.034, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "missed": [], + "recall_at_5": 0.75, + "precision_at_5": 0.6, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 53303, + "latency_ms": 130.07, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "missed": [ + "lib/error-handler.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 139650, + "latency_ms": 215.632, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [ + "lib/handleRequest.js" + ], + "missed": [ + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 78858, + "latency_ms": 134.383, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/router-options.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 114663, + "latency_ms": 160.093, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86593, + "latency_ms": 137.927, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 104673, + "latency_ms": 134.926, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/contentTypeParser.js", + "lib/errors.js" + ], + "missed": [ + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 111702, + "latency_ms": 106.697, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 102629, + "latency_ms": 155.032, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [ + "test/types/route.test-d.ts" + ], + "missed": [ + "types/route.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 123063, + "latency_ms": 133.395, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js" + ], + "missed": [ + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0.6, + "precision_at_5": 0.6, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 1, + "tokens_delivered": 130397, + "latency_ms": 162.707, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/decorate.js", + "lib/reply.js" + ], + "missed": [ + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 119132, + "latency_ms": 170.734, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [ + "lib/validation.js" + ], + "missed": [ + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 140649, + "latency_ms": 150.666, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 111682, + "latency_ms": 139.688, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53367, + "latency_ms": 111.4, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 125116, + "latency_ms": 138.499, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/throw.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 84828, + "latency_ms": 139.556, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [ + "lib/head-route.js", + "test/route.6.test.js" + ], + "missed": [ + "test/route.7.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 120657, + "latency_ms": 133.603, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [ + "lib/request.js", + "test/server.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 61362, + "latency_ms": 112.749, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71679, + "latency_ms": 136.612, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 132990, + "latency_ms": 127.809, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "missed": [ + "test/issue-4959.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.8333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 1, + "tokens_delivered": 108175, + "latency_ms": 148.007, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 96459, + "latency_ms": 140.331, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js", + "test/close.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 136620, + "latency_ms": 136.98, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 47.345, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.147, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.439, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.613, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 43.749, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.192, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.449, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 44.929, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.872, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 48.362, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 43.741, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 44.033, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.04, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.924, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [], + "missed": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 47.274, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 49.343, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.834, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.718, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 48.369, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 48.038, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 47.34, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.747, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.723, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 53.111, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 53.199, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 51.133, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 56.524, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 44.694, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.345, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 47.718, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 53.935, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 48.468, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.288, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 51.735, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 43.895, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/find-route.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 58.96, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 44.122, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.23, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.827, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.211, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 54.055, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.49, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.002, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/404s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.43, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.639, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.796, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.484, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.893, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 47.05, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.907, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.342, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.65, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 47.117, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 47.168, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.916, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 49.094, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 44.123, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 61.329, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 47.507, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 47.172, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 48.635, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.441, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 44.812, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.994, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 51.53, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 44.522, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 54.366, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.329, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.863, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.801, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.641, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 47.718, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.378, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 49.128, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 46.774, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 56.471, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 51.977, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.321, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 47.366, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [], + "missed": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.972, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.206, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 47.465, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 73281, + "latency_ms": 326.215, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 60462, + "latency_ms": 152.98, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "fastify.js", + "test/http-methods/custom-http-methods.test.js", + "types/instance.d.ts" + ], + "missed": [ + "lib/warnings.js", + "test/types/instance.tst.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 5, + "tokens_delivered": 53141, + "latency_ms": 209.07, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts" + ], + "missed": [ + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 24682, + "latency_ms": 287.46, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1084, + "latency_ms": 152.927, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6718, + "latency_ms": 192.169, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51446, + "latency_ms": 248.182, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js" + ], + "missed": [ + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 18036, + "latency_ms": 347.744, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49127, + "latency_ms": 241.241, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/request-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 53872, + "latency_ms": 219.87, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "fastify.d.ts", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 49971, + "latency_ms": 215.314, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js" + ], + "missed": [ + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 54500, + "latency_ms": 455.919, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 63374, + "latency_ms": 279.137, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/decorate.js" + ], + "missed": [ + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 40544, + "latency_ms": 305.085, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [ + "test/internals/reply.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 27257, + "latency_ms": 317.433, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "lib/errors.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 77397, + "latency_ms": 231.811, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8030, + "latency_ms": 189.313, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 85122, + "latency_ms": 229.713, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10389, + "latency_ms": 197.51, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1695, + "latency_ms": 264.18, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27676, + "latency_ms": 233.652, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 61933, + "latency_ms": 160.914, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [ + "fastify.js", + "lib/server.js", + "test/http2/closing.test.js" + ], + "missed": [ + "lib/symbols.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 41006, + "latency_ms": 180.501, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 83879, + "latency_ms": 260.088, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18231, + "latency_ms": 116.968, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19096, + "latency_ms": 179.425, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 68263, + "latency_ms": 270.609, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68496, + "latency_ms": 219.116, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49480, + "latency_ms": 265.293, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8661, + "latency_ms": 304.028, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49904, + "latency_ms": 296.104, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 226.1, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [ + "test/helper.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 61377, + "latency_ms": 222.539, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10237, + "latency_ms": 233.352, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22572, + "latency_ms": 187.759, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js", + "test/find-route.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 89664, + "latency_ms": 194.319, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 110.671, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 61951, + "latency_ms": 197.514, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [ + "lib/handle-request.js" + ], + "missed": [ + "lib/error-handler.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 4, + "tokens_delivered": 39777, + "latency_ms": 268.482, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 57191, + "latency_ms": 160.351, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45748, + "latency_ms": 313.531, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 10361, + "latency_ms": 236.418, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 71134, + "latency_ms": 157.325, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/404s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36724, + "latency_ms": 189.052, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6523, + "latency_ms": 310.624, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69553, + "latency_ms": 281.143, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [ + "types/logger.d.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 26803, + "latency_ms": 226.113, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8911, + "latency_ms": 158.984, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36728, + "latency_ms": 340.632, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21715, + "latency_ms": 330.113, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14723, + "latency_ms": 348.676, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 196.964, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7648, + "latency_ms": 200.299, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [ + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 34399, + "latency_ms": 261.786, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 254.767, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 11328, + "latency_ms": 289.97, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 36646, + "latency_ms": 335.385, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/fastify-instance.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 35621, + "latency_ms": 283.383, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [ + "types/logger.d.ts" + ], + "missed": [ + "lib/log-controller.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 47231, + "latency_ms": 335.636, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 11970, + "latency_ms": 243.809, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "lib/error-handler.js", + "types/instance.d.ts" + ], + "missed": [ + "test/500s.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 60322, + "latency_ms": 353.447, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49199, + "latency_ms": 203.029, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 55458, + "latency_ms": 303.54, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [ + "types/reply.d.ts" + ], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 49489, + "latency_ms": 282.548, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45132, + "latency_ms": 272.066, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [ + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 50285, + "latency_ms": 154.896, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 351.661, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45744, + "latency_ms": 270.576, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/plugin-utils.js" + ], + "missed": [ + "lib/errors.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 55459, + "latency_ms": 342.826, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/reply.js", + "lib/request.js" + ], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 42080, + "latency_ms": 362.528, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [ + "lib/validation.js" + ], + "missed": [ + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 81940, + "latency_ms": 344.929, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 7166, + "latency_ms": 269.861, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34882, + "latency_ms": 160.735, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43893, + "latency_ms": 268.83, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 65065, + "latency_ms": 228.141, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [ + "lib/head-route.js" + ], + "missed": [ + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 46052, + "latency_ms": 240.171, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3154, + "latency_ms": 158.549, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 85760, + "latency_ms": 273.58, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 69662, + "latency_ms": 157.495, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "fastify.d.ts" + ], + "missed": [ + "build/build-validation.js", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 1, + "tokens_delivered": 79519, + "latency_ms": 304.984, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10034, + "latency_ms": 270.037, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 94929, + "latency_ms": 283.744, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18638, + "latency_ms": 12.973, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2709, + "latency_ms": 11.59, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "test/http-methods/custom-http-methods.test.js" + ], + "missed": [ + "fastify.js", + "lib/warnings.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 5, + "tokens_delivered": 14347, + "latency_ms": 11.628, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23602, + "latency_ms": 11.712, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12986, + "latency_ms": 10.877, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [ + "fastify.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 17294, + "latency_ms": 11.098, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40878, + "latency_ms": 10.568, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 27324, + "latency_ms": 11.204, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31441, + "latency_ms": 11.265, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [ + "lib/request.js", + "test/request-error.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 32044, + "latency_ms": 11.093, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42413, + "latency_ms": 9.952, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js" + ], + "missed": [ + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1111, + "recall_at_20": 0.3333, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 17801, + "latency_ms": 10.233, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3048, + "latency_ms": 10.772, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "test/decorator-instance-properties.test.js" + ], + "missed": [ + "lib/decorate.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 10172, + "latency_ms": 10.551, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [ + "test/internals/reply.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 26634, + "latency_ms": 11.045, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "lib/errors.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 45225, + "latency_ms": 11.544, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.929, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14066, + "latency_ms": 10.716, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 56156, + "latency_ms": 11.351, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.34, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8563, + "latency_ms": 11.07, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 34446, + "latency_ms": 10.539, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [ + "lib/server.js", + "test/http2/closing.test.js" + ], + "missed": [ + "fastify.js", + "lib/symbols.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 17422, + "latency_ms": 11.849, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 34433, + "latency_ms": 10.997, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.529, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 6.116, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 6.106, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5188, + "latency_ms": 11.245, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 6.383, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "test/trust-proxy.test.js" + ], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3581, + "latency_ms": 10.917, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14619, + "latency_ms": 10.246, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 476, + "latency_ms": 10.15, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16143, + "latency_ms": 9.636, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 31010, + "latency_ms": 10.37, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [ + "test/close-pipelining.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 36233, + "latency_ms": 10.476, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js", + "test/find-route.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 38692, + "latency_ms": 10.506, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.262, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 33213, + "latency_ms": 11.185, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [ + "lib/error-handler.js" + ], + "missed": [ + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 7, + "tokens_delivered": 28195, + "latency_ms": 10.459, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 38008, + "latency_ms": 10.91, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/reply.js" + ], + "missed": [ + "lib/handle-request.js", + "lib/request.js" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 1, + "tokens_delivered": 39766, + "latency_ms": 10.448, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "test/reply-trailers.test.js" + ], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3553, + "latency_ms": 9.861, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [ + "test/hooks-async.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 16591, + "latency_ms": 9.836, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/404s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.589, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/route.js", + "test/hooks.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 48445, + "latency_ms": 10.933, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 45998, + "latency_ms": 11.323, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16823, + "latency_ms": 10.599, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35580, + "latency_ms": 10.625, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3487, + "latency_ms": 10.02, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5434, + "latency_ms": 9.556, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "test/trust-proxy.test.js" + ], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 39341, + "latency_ms": 10.549, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.452, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.12, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [ + "lib/error-handler.js" + ], + "missed": [ + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 12, + "tokens_delivered": 17815, + "latency_ms": 11.045, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "test/close.test.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 5094, + "latency_ms": 10.832, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [ + "types/request.d.ts" + ], + "missed": [ + "test/types/request.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 37664, + "latency_ms": 11.488, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 27059, + "latency_ms": 10.898, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2494, + "latency_ms": 10.351, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33476, + "latency_ms": 11.015, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "test/trust-proxy.test.js" + ], + "missed": [ + "fastify.d.ts", + "lib/request.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 1, + "recall_at_10": 0.25, + "precision_at_10": 1, + "recall_at_20": 0.25, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 2835, + "latency_ms": 10.731, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "lib/error-handler.js" + ], + "missed": [ + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 33290, + "latency_ms": 10.696, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36722, + "latency_ms": 11.847, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [ + "test/router-options.test.js" + ], + "missed": [ + "lib/route.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 11008, + "latency_ms": 10.595, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [ + "test/types/reply.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/type-provider.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 48763, + "latency_ms": 11.628, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29434, + "latency_ms": 11.737, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1408, + "latency_ms": 11.741, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [ + "test/close-pipelining.test.js" + ], + "missed": [ + "fastify.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 5098, + "latency_ms": 11.003, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 59832, + "latency_ms": 10.779, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/plugin-utils.js" + ], + "missed": [ + "lib/errors.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 8, + "tokens_delivered": 24505, + "latency_ms": 11.199, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/reply.js", + "lib/request.js" + ], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 44360, + "latency_ms": 10.888, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 29747, + "latency_ms": 11.471, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3809, + "latency_ms": 11.395, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.412, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14722, + "latency_ms": 10.725, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26835, + "latency_ms": 10.972, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 29594, + "latency_ms": 11.918, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.904, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 38197, + "latency_ms": 11.765, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2317, + "latency_ms": 11.434, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "test/router-options.test.js" + ], + "missed": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 1, + "tokens_delivered": 41079, + "latency_ms": 11.964, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22478, + "latency_ms": 12.81, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "test/close.test.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 26049, + "latency_ms": 11.607, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/M-fastify-ret.json b/benchmarks/context-retrieval/results/M-fastify-ret.json new file mode 100644 index 00000000..5e926c2d --- /dev/null +++ b/benchmarks/context-retrieval/results/M-fastify-ret.json @@ -0,0 +1,12150 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "fastify", + "head": "83e69762aff047940b376fc2a266d143499ef727" + }, + "tier": "medium", + "tier_code_files": 300, + "corpus_counts": { + "cases": 82, + "path_leak": 47, + "baseline_blind": 35, + "multi_file": 66 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.7604, + "control": 0, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "replace AssertionError with FST_ERR_PLUGIN_DEPENDENCY_NOT_REGISTERED in checkDependencies (#6774)", + "revision": "27f4d6222dd945238e96c1fdaa632d5dc3c37b2c", + "returned": [ + ".github/workflows/coverage-nix.yml", + "examples/route-prefix.js", + "test/internals/hooks.test.js", + "lib/content-type.js", + "test/router-options.test.js" + ] + }, + { + "provider_id": "churn-ranked", + "query": "add http method override warning (#6879)", + "revision": "b93d611c7cefc2cd4a0b6f05cc864f2b6c397a6d", + "returned": [ + "package.json", + "fastify.js", + ".github/workflows/ci.yml", + "lib/reply.js", + ".github/workflows/links-check.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "use JSON.stringify in onBadUrl for proper escaping (#6420)", + "revision": "970c575832521fff01cc018d928d84454811173b", + "returned": [ + "package.json", + "fastify.js", + ".github/workflows/ci.yml", + "lib/reply.js", + "lib/route.js" + ] + }, + { + "provider_id": "churn-ranked", + "query": "run preClose hook exactly once per declaring instance (#6940)", + "revision": "9a1535bcfe2a780cc6f9c6555df86e5072ca9a29", + "returned": [ + "package.json", + "fastify.js", + ".github/workflows/ci.yml", + "lib/reply.js", + "fastify.d.ts" + ] + }, + { + "provider_id": "semble", + "query": "uncatchable throws in writeHead when using async hook (#6881)", + "revision": "b6cdc6a40598d15f38fe7d399eb17f4b84b7bf06", + "returned": [ + "lib/reply.js", + "lib/hooks.js", + "lib/error-handler.js", + "lib/log-controller.js", + "fastify.js" + ] + }, + { + "provider_id": "semble", + "query": "set status code before publishing diagnostics error channel (#6412)", + "revision": "ab9dae462a138cd0d3c2d4e4e9e691b426fbbf9e", + "returned": [ + "lib/error-handler.js", + "lib/wrap-thenable.js", + "fastify.js", + "lib/handle-request.js", + "lib/reply.js" + ] + }, + { + "provider_id": "jcodemunch", + "query": "allow explicit http2: false in server options (#6888)", + "revision": "0e9a3b35d8b13c9f261bf29bef46168a5d43c28f", + "returned": [ + "fastify.d.ts", + "test/types/register.tst.ts" + ] + }, + { + "provider_id": "jcodemunch", + "query": "internal function _addHook failure should be turned into the rejection that app.ready() is waiting for (#6105)", + "revision": "850ebf6ef0d71284e37c2bc98f41bbe2ec2c45c0", + "returned": [ + "fastify.js" + ] + }, + { + "provider_id": "jcodemunch", + "query": "run preClose hook exactly once per declaring instance (#6940)", + "revision": "9a1535bcfe2a780cc6f9c6555df86e5072ca9a29", + "returned": [ + "test/types/hooks.tst.ts", + "types/hooks.d.ts", + "test/close.test.js" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "add http method override warning (#6879)", + "revision": "b93d611c7cefc2cd4a0b6f05cc864f2b6c397a6d", + "returned": [ + "docs/Reference/Server.md", + "fastify.d.ts", + "types/route.d.ts", + "types/instance.d.ts", + "docs/Guides/Migration-Guide-V5.md" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0024, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0329, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0091, + "mean_precision_at_10": 0.0012, + "zero_hit_rate": 0.9146, + "median_tokens_delivered": 30284.5, + "median_latency_ms": 13.0255, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0024, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0329, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0091, + "mean_precision_at_10": 0.0012, + "zero_hit_rate": 0.9146, + "median_tokens_delivered": 30284.5, + "median_latency_ms": 13.0255, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0043, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0255, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0053, + "mean_precision_at_10": 0.0021, + "zero_hit_rate": 0.9149, + "median_tokens_delivered": 32313, + "median_latency_ms": 12.832, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0429, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0143, + "mean_precision_at_10": 0, + "zero_hit_rate": 0.9143, + "median_tokens_delivered": 29377, + "median_latency_ms": 13.074, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 15272.5, + "peak_rss_mb": 15348, + "delta_rss_mb": 75.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.1461, + "full_recall_at_5": 0.0366, + "mean_recall_at_10": 0.2191, + "full_recall_at_10": 0.0366, + "mean_recall_at_20": 0.3053, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1187, + "mean_precision_at_10": 0.0488, + "zero_hit_rate": 0.439, + "median_tokens_delivered": 70828, + "median_latency_ms": 28.6, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.1461, + "full_recall_at_5": 0.0366, + "mean_recall_at_10": 0.2191, + "full_recall_at_10": 0.0366, + "mean_recall_at_20": 0.3053, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1187, + "mean_precision_at_10": 0.0488, + "zero_hit_rate": 0.439, + "median_tokens_delivered": 70828, + "median_latency_ms": 28.6, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.1344, + "full_recall_at_5": 0.0213, + "mean_recall_at_10": 0.2085, + "full_recall_at_10": 0.0213, + "mean_recall_at_20": 0.2986, + "full_recall_at_20": 0.0426, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1184, + "mean_precision_at_10": 0.0532, + "zero_hit_rate": 0.4043, + "median_tokens_delivered": 71582, + "median_latency_ms": 28.673, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.1619, + "full_recall_at_5": 0.0571, + "mean_recall_at_10": 0.2333, + "full_recall_at_10": 0.0571, + "mean_recall_at_20": 0.3143, + "full_recall_at_20": 0.1143, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.119, + "mean_precision_at_10": 0.0429, + "zero_hit_rate": 0.4857, + "median_tokens_delivered": 70715, + "median_latency_ms": 28.076, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 15362.8, + "peak_rss_mb": 15680.8, + "delta_rss_mb": 317.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.5207, + "full_recall_at_5": 0.2195, + "mean_recall_at_10": 0.676, + "full_recall_at_10": 0.4146, + "mean_recall_at_20": 0.7604, + "full_recall_at_20": 0.5488, + "mean_recall_at_1000_tokens": 0.5146, + "mean_recall_at_4000_tokens": 0.7604, + "mean_recall_at_16000_tokens": 0.7604, + "mean_precision_at_10": 0.1512, + "zero_hit_rate": 0.0854, + "median_tokens_delivered": 3631.5, + "median_latency_ms": 1441.9945, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.5207, + "full_recall_at_5": 0.2195, + "mean_recall_at_10": 0.676, + "full_recall_at_10": 0.4146, + "mean_recall_at_20": 0.7604, + "full_recall_at_20": 0.5488, + "mean_recall_at_1000_tokens": 0.5146, + "mean_recall_at_4000_tokens": 0.7604, + "mean_recall_at_16000_tokens": 0.7604, + "mean_precision_at_10": 0.1512, + "zero_hit_rate": 0.0854, + "median_tokens_delivered": 3631.5, + "median_latency_ms": 1441.9945, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.5766, + "full_recall_at_5": 0.234, + "mean_recall_at_10": 0.7333, + "full_recall_at_10": 0.4255, + "mean_recall_at_20": 0.8415, + "full_recall_at_20": 0.617, + "mean_recall_at_1000_tokens": 0.5766, + "mean_recall_at_4000_tokens": 0.8415, + "mean_recall_at_16000_tokens": 0.8415, + "mean_precision_at_10": 0.1787, + "zero_hit_rate": 0.0213, + "median_tokens_delivered": 3617, + "median_latency_ms": 1441.783, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.4457, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.599, + "full_recall_at_10": 0.4, + "mean_recall_at_20": 0.6514, + "full_recall_at_20": 0.4571, + "mean_recall_at_1000_tokens": 0.4314, + "mean_recall_at_4000_tokens": 0.6514, + "mean_recall_at_16000_tokens": 0.6514, + "mean_precision_at_10": 0.1143, + "zero_hit_rate": 0.1714, + "median_tokens_delivered": 3648, + "median_latency_ms": 1445.952, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 15266.9, + "peak_rss_mb": 39065.3, + "delta_rss_mb": 23798.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "jcodemunch", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.2372, + "full_recall_at_5": 0.0854, + "mean_recall_at_10": 0.2372, + "full_recall_at_10": 0.0854, + "mean_recall_at_20": 0.2372, + "full_recall_at_20": 0.0854, + "mean_recall_at_1000_tokens": 0.223, + "mean_recall_at_4000_tokens": 0.2372, + "mean_recall_at_16000_tokens": 0.2372, + "mean_precision_at_10": 0.2154, + "zero_hit_rate": 0.5854, + "median_tokens_delivered": 723.5, + "median_latency_ms": 453.0215, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 10 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.2372, + "full_recall_at_5": 0.0854, + "mean_recall_at_10": 0.2372, + "full_recall_at_10": 0.0854, + "mean_recall_at_20": 0.2372, + "full_recall_at_20": 0.0854, + "mean_recall_at_1000_tokens": 0.223, + "mean_recall_at_4000_tokens": 0.2372, + "mean_recall_at_16000_tokens": 0.2372, + "mean_precision_at_10": 0.2154, + "zero_hit_rate": 0.5854, + "median_tokens_delivered": 723.5, + "median_latency_ms": 453.0215, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 10 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.2684, + "full_recall_at_5": 0.1064, + "mean_recall_at_10": 0.2684, + "full_recall_at_10": 0.1064, + "mean_recall_at_20": 0.2684, + "full_recall_at_20": 0.1064, + "mean_recall_at_1000_tokens": 0.2543, + "mean_recall_at_4000_tokens": 0.2684, + "mean_recall_at_16000_tokens": 0.2684, + "mean_precision_at_10": 0.2571, + "zero_hit_rate": 0.5319, + "median_tokens_delivered": 795, + "median_latency_ms": 457.125, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 8 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.1952, + "full_recall_at_5": 0.0571, + "mean_recall_at_10": 0.1952, + "full_recall_at_10": 0.0571, + "mean_recall_at_20": 0.1952, + "full_recall_at_20": 0.0571, + "mean_recall_at_1000_tokens": 0.181, + "mean_recall_at_4000_tokens": 0.1952, + "mean_recall_at_16000_tokens": 0.1952, + "mean_precision_at_10": 0.1595, + "zero_hit_rate": 0.6571, + "median_tokens_delivered": 650, + "median_latency_ms": 445.118, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 2 + } + }, + "outcomes": { + "answered": 72, + "indexed-but-returned-nothing": 10 + }, + "resources": { + "baseline_rss_mb": 17720.4, + "peak_rss_mb": 42648.6, + "delta_rss_mb": 24928.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "cocoindex-code", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.0882, + "full_recall_at_5": 0.0122, + "mean_recall_at_10": 0.1224, + "full_recall_at_10": 0.0488, + "mean_recall_at_20": 0.1437, + "full_recall_at_20": 0.0854, + "mean_recall_at_1000_tokens": 0.0167, + "mean_recall_at_4000_tokens": 0.1089, + "mean_recall_at_16000_tokens": 0.1437, + "mean_precision_at_10": 0.0378, + "zero_hit_rate": 0.7683, + "median_tokens_delivered": 4854.5, + "median_latency_ms": 974.146, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.0882, + "full_recall_at_5": 0.0122, + "mean_recall_at_10": 0.1224, + "full_recall_at_10": 0.0488, + "mean_recall_at_20": 0.1437, + "full_recall_at_20": 0.0854, + "mean_recall_at_1000_tokens": 0.0167, + "mean_recall_at_4000_tokens": 0.1089, + "mean_recall_at_16000_tokens": 0.1437, + "mean_precision_at_10": 0.0378, + "zero_hit_rate": 0.7683, + "median_tokens_delivered": 4854.5, + "median_latency_ms": 974.146, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.1092, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1369, + "full_recall_at_10": 0.0426, + "mean_recall_at_20": 0.1741, + "full_recall_at_20": 0.1064, + "mean_recall_at_1000_tokens": 0.0291, + "mean_recall_at_4000_tokens": 0.1135, + "mean_recall_at_16000_tokens": 0.1741, + "mean_precision_at_10": 0.0484, + "zero_hit_rate": 0.7021, + "median_tokens_delivered": 4851, + "median_latency_ms": 994.498, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.06, + "full_recall_at_5": 0.0286, + "mean_recall_at_10": 0.1029, + "full_recall_at_10": 0.0571, + "mean_recall_at_20": 0.1029, + "full_recall_at_20": 0.0571, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1029, + "mean_recall_at_16000_tokens": 0.1029, + "mean_precision_at_10": 0.0237, + "zero_hit_rate": 0.8571, + "median_tokens_delivered": 4870, + "median_latency_ms": 967.462, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 39821.8, + "peak_rss_mb": 47170.2, + "delta_rss_mb": 7348.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55380, + "latency_ms": 33.265, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14735, + "latency_ms": 16.436, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33582, + "latency_ms": 13.828, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40619, + "latency_ms": 14.254, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34349, + "latency_ms": 14.082, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49787, + "latency_ms": 15.048, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19564, + "latency_ms": 16.456, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22900, + "latency_ms": 14.304, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [ + "test/types/request.test-d.ts" + ], + "missed": [ + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 29021, + "latency_ms": 12.456, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42213, + "latency_ms": 12.031, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25050, + "latency_ms": 12.17, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47319, + "latency_ms": 14.147, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 25876, + "latency_ms": 13.584, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 52479, + "latency_ms": 12.305, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [], + "missed": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14502, + "latency_ms": 13.468, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js" + ], + "missed": [ + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 15, + "tokens_delivered": 18095, + "latency_ms": 15.218, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29377, + "latency_ms": 13.598, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31292, + "latency_ms": 13.846, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45307, + "latency_ms": 13.91, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18048, + "latency_ms": 13.033, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28081, + "latency_ms": 12.478, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29389, + "latency_ms": 13.053, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49530, + "latency_ms": 13.023, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28931, + "latency_ms": 13.101, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23861, + "latency_ms": 12.234, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30703, + "latency_ms": 12.935, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28832, + "latency_ms": 13.786, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19751, + "latency_ms": 14.712, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 38242, + "latency_ms": 13.074, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33259, + "latency_ms": 12.732, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29737, + "latency_ms": 12.559, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30714, + "latency_ms": 11.956, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26147, + "latency_ms": 13.235, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23070, + "latency_ms": 13.195, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25310, + "latency_ms": 13.888, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/find-route.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33788, + "latency_ms": 13.493, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31982, + "latency_ms": 13.758, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10713, + "latency_ms": 14.199, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18276, + "latency_ms": 12.749, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30719, + "latency_ms": 13.717, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34315, + "latency_ms": 12.065, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42597, + "latency_ms": 12.158, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32783, + "latency_ms": 13.041, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/404s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14947, + "latency_ms": 13.15, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22479, + "latency_ms": 12.58, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30757, + "latency_ms": 12.823, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30239, + "latency_ms": 14.147, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18963, + "latency_ms": 14.883, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "test/hooks.test.js" + ], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 56605, + "latency_ms": 13.758, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29992, + "latency_ms": 12.138, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28153, + "latency_ms": 12.388, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30262, + "latency_ms": 12.546, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45885, + "latency_ms": 12.391, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23215, + "latency_ms": 11.901, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36564, + "latency_ms": 11.662, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46897, + "latency_ms": 13.737, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32459, + "latency_ms": 13.464, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22393, + "latency_ms": 11.967, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28957, + "latency_ms": 12.315, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51533, + "latency_ms": 12.297, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30177, + "latency_ms": 12.832, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42868, + "latency_ms": 12.074, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34695, + "latency_ms": 11.742, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40034, + "latency_ms": 12.5, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46434, + "latency_ms": 12.357, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36331, + "latency_ms": 12.486, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30307, + "latency_ms": 11.942, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40300, + "latency_ms": 11.414, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/plugin-utils.js" + ], + "missed": [ + "lib/errors.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 32313, + "latency_ms": 14.923, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/decorate.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 28910, + "latency_ms": 15.625, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43992, + "latency_ms": 13.435, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23870, + "latency_ms": 12.705, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42687, + "latency_ms": 12.07, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26329, + "latency_ms": 12.918, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28075, + "latency_ms": 13.5, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46278, + "latency_ms": 13.403, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38893, + "latency_ms": 12.463, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22541, + "latency_ms": 12.785, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45100, + "latency_ms": 12.193, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [], + "missed": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17721, + "latency_ms": 11.577, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26451, + "latency_ms": 11.771, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24066, + "latency_ms": 13.028, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70557, + "latency_ms": 28.833, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82986, + "latency_ms": 34.835, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "fastify.js", + "types/instance.d.ts" + ], + "missed": [ + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 70557, + "latency_ms": 32.702, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts" + ], + "missed": [ + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 70629, + "latency_ms": 27.368, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 87404, + "latency_ms": 29.145, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [ + "fastify.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 63466, + "latency_ms": 27.872, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70776, + "latency_ms": 27.843, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70662, + "latency_ms": 25.539, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69669, + "latency_ms": 28.545, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/request-error.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 66533, + "latency_ms": 27.841, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "fastify.js" + ], + "missed": [ + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 65423, + "latency_ms": 27.343, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 71928, + "latency_ms": 28.501, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 74650, + "latency_ms": 29.603, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70662, + "latency_ms": 26.619, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [], + "missed": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70557, + "latency_ms": 26.974, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "lib/route.js" + ], + "missed": [ + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 74728, + "latency_ms": 27.606, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69389, + "latency_ms": 28.076, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/http2/plain.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 71207, + "latency_ms": 27.112, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82986, + "latency_ms": 28.862, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64300, + "latency_ms": 27.123, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70546, + "latency_ms": 27.255, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 89027, + "latency_ms": 30.769, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [ + "fastify.js", + "lib/server.js" + ], + "missed": [ + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 82521, + "latency_ms": 30.64, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 80366, + "latency_ms": 31.72, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 74508, + "latency_ms": 27.459, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 64846, + "latency_ms": 29.974, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74429, + "latency_ms": 26.35, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/errors.js", + "test/web-api.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 84523, + "latency_ms": 30.432, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71253, + "latency_ms": 26.934, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 74448, + "latency_ms": 28.086, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 82525, + "latency_ms": 28.606, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62953, + "latency_ms": 30.009, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63778, + "latency_ms": 27.931, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 60591, + "latency_ms": 29.665, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63424, + "latency_ms": 30.651, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/find-route.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 63892, + "latency_ms": 27.092, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80545, + "latency_ms": 28.075, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 63909, + "latency_ms": 28.083, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68086, + "latency_ms": 27.352, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 84613, + "latency_ms": 27.891, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/reply.js", + "lib/request.js" + ], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 5, + "tokens_delivered": 63724, + "latency_ms": 28.901, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 70641, + "latency_ms": 29.462, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 61937, + "latency_ms": 28.723, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/404s.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 72316, + "latency_ms": 27.902, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "lib/route.js" + ], + "missed": [ + "test/hooks.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 63201, + "latency_ms": 29.39, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82525, + "latency_ms": 30.796, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64345, + "latency_ms": 31.441, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54068, + "latency_ms": 31.461, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 62953, + "latency_ms": 26.951, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/internals/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 69439, + "latency_ms": 27.876, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 74456, + "latency_ms": 29.785, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 68086, + "latency_ms": 26.271, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84659, + "latency_ms": 29.599, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70880, + "latency_ms": 30.512, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 70638, + "latency_ms": 27.092, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74650, + "latency_ms": 25.675, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63886, + "latency_ms": 28.522, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/fastify-instance.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 86006, + "latency_ms": 30.192, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70662, + "latency_ms": 26.777, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 84631, + "latency_ms": 28.238, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 89074, + "latency_ms": 28.952, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64345, + "latency_ms": 28.594, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 72779, + "latency_ms": 28.652, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts" + ], + "missed": [ + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 65418, + "latency_ms": 29.286, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74997, + "latency_ms": 26.376, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63778, + "latency_ms": 31.229, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 74705, + "latency_ms": 27.726, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 72798, + "latency_ms": 27.714, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js" + ], + "missed": [ + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 71582, + "latency_ms": 27.981, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/reply.js", + "lib/request.js" + ], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 5, + "tokens_delivered": 63025, + "latency_ms": 31.185, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 72300, + "latency_ms": 29.396, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 74694, + "latency_ms": 26.381, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64846, + "latency_ms": 30.189, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74728, + "latency_ms": 27.668, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 74702, + "latency_ms": 50.174, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 72267, + "latency_ms": 72.524, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 72267, + "latency_ms": 50.476, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 84982, + "latency_ms": 38.951, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 70715, + "latency_ms": 31.117, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "test/types/instance.test-d.ts" + ], + "missed": [ + "build/build-validation.js", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 65361, + "latency_ms": 28.673, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84594, + "latency_ms": 32.598, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 84607, + "latency_ms": 29.994, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [ + "lib/hooks.js" + ], + "missed": [ + "test/reply-error.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3468, + "latency_ms": 1863.201, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3483, + "latency_ms": 1288.149, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "types/instance.d.ts" + ], + "missed": [ + "test/types/instance.tst.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.8, + "precision_at_10": 0.4, + "recall_at_20": 0.8, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.4, + "recall_at_4000_tokens": 0.8, + "recall_at_16000_tokens": 0.8, + "first_hit_rank": 1, + "tokens_delivered": 3513, + "latency_ms": 1403.573, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3721, + "latency_ms": 1379.492, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3415, + "latency_ms": 1485.181, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [ + "fastify.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3467, + "latency_ms": 2408.129, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3993, + "latency_ms": 1417.104, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3645, + "latency_ms": 1423.657, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [ + "types/request.d.ts" + ], + "missed": [ + "test/types/request.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 3717, + "latency_ms": 1324.493, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [ + "lib/request.js", + "test/request-error.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3359, + "latency_ms": 1305.394, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3674, + "latency_ms": 1296.936, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3859, + "latency_ms": 1354.76, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3571, + "latency_ms": 1418.478, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3886, + "latency_ms": 1442.206, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [ + "test/internals/reply.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3764, + "latency_ms": 1410.065, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "lib/route.js" + ], + "missed": [ + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3919, + "latency_ms": 1402.431, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3706, + "latency_ms": 1321.848, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/http2/plain.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3769, + "latency_ms": 3039.985, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3785, + "latency_ms": 1287.164, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3356, + "latency_ms": 1389.053, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [ + "lib/validation.js" + ], + "missed": [ + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 3465, + "latency_ms": 1429.621, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3507, + "latency_ms": 1412.189, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [ + "fastify.js", + "lib/server.js", + "test/http2/closing.test.js" + ], + "missed": [ + "lib/symbols.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.1579, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 3567, + "latency_ms": 1412.363, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4147, + "latency_ms": 1315.527, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/request-port.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3150, + "latency_ms": 1395.25, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3773, + "latency_ms": 1364.965, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3641, + "latency_ms": 1505.063, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.1579, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3538, + "latency_ms": 1656.004, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "fastify.js", + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3556, + "latency_ms": 2509.227, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3563, + "latency_ms": 1380.968, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3781, + "latency_ms": 1790.823, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3617, + "latency_ms": 2224.469, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [ + "test/helper.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 3132, + "latency_ms": 1282.548, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [ + "fastify.js", + "lib/server.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3711, + "latency_ms": 1347.071, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [ + "test/close-pipelining.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3349, + "latency_ms": 1314.224, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js", + "test/find-route.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3829, + "latency_ms": 1401.872, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3648, + "latency_ms": 1313.041, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3721, + "latency_ms": 1488.092, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [ + "lib/error-handler.js", + "lib/handle-request.js" + ], + "missed": [ + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3296, + "latency_ms": 1438.528, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3857, + "latency_ms": 1450.107, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js" + ], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0.6, + "precision_at_5": 0.6, + "recall_at_10": 0.8, + "precision_at_10": 0.4, + "recall_at_20": 0.8, + "precision_at_20": 0.2353, + "recall_at_1000_tokens": 0.6, + "recall_at_4000_tokens": 0.8, + "recall_at_16000_tokens": 0.8, + "first_hit_rank": 1, + "tokens_delivered": 3783, + "latency_ms": 1494.024, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3723, + "latency_ms": 1441.783, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [ + "test/hooks-async.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 3705, + "latency_ms": 2613.056, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/404s.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3915, + "latency_ms": 1445.952, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4007, + "latency_ms": 1464.914, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3298, + "latency_ms": 1354.904, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [ + "types/logger.d.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3864, + "latency_ms": 1318.784, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3423, + "latency_ms": 1361.064, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 3721, + "latency_ms": 1390.572, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/internals/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3701, + "latency_ms": 1354.427, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3718, + "latency_ms": 1477.197, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3765, + "latency_ms": 1663.955, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3537, + "latency_ms": 1395.561, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "missed": [ + "lib/error-status.js" + ], + "recall_at_5": 0.75, + "precision_at_5": 0.6, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 3111, + "latency_ms": 1936.37, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js", + "test/close.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3521, + "latency_ms": 1628.087, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3865, + "latency_ms": 1461.282, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3727, + "latency_ms": 1388.875, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/fastify-instance.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3696, + "latency_ms": 1592.078, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3622, + "latency_ms": 1653.253, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [ + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0.75, + "precision_at_5": 0.6, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 3056, + "latency_ms": 1613.631, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "lib/error-handler.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "missed": [ + "test/500s.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 2, + "tokens_delivered": 3838, + "latency_ms": 1472.255, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [ + "lib/handleRequest.js" + ], + "missed": [ + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3526, + "latency_ms": 1561.942, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [ + "lib/route.js", + "test/router-options.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3480, + "latency_ms": 2956.996, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [ + "test/types/reply.test-d.ts", + "types/type-provider.d.ts" + ], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 3, + "tokens_delivered": 3526, + "latency_ms": 1510.139, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [ + "lib/schemas.js" + ], + "missed": [ + "test/schema-serialization.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3529, + "latency_ms": 2571.848, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "missed": [ + "test/buffer.test.js" + ], + "recall_at_5": 0.6, + "precision_at_5": 0.6, + "recall_at_10": 0.8, + "precision_at_10": 0.4, + "recall_at_20": 0.8, + "precision_at_20": 0.2353, + "recall_at_1000_tokens": 0.6, + "recall_at_4000_tokens": 0.8, + "recall_at_16000_tokens": 0.8, + "first_hit_rank": 1, + "tokens_delivered": 3750, + "latency_ms": 1842.918, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3593, + "latency_ms": 1797.86, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3620, + "latency_ms": 1690.992, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "types/errors.d.ts" + ], + "missed": [ + "test/types/errors.test-d.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.8, + "precision_at_10": 0.4, + "recall_at_20": 0.8, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.4, + "recall_at_4000_tokens": 0.8, + "recall_at_16000_tokens": 0.8, + "first_hit_rank": 1, + "tokens_delivered": 3561, + "latency_ms": 1617.455, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js" + ], + "missed": [ + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 2, + "tokens_delivered": 3541, + "latency_ms": 1620.491, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3261, + "latency_ms": 1974.977, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3617, + "latency_ms": 1932.159, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2867, + "latency_ms": 3690.475, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [ + "lib/schema-controller.js" + ], + "missed": [ + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3823, + "latency_ms": 5530.361, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 3451, + "latency_ms": 2832.846, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3872, + "latency_ms": 2225.699, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 3451, + "latency_ms": 1305.723, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3764, + "latency_ms": 1283.08, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3668, + "latency_ms": 1931.099, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "lib/config-validator.js", + "test/router-options.test.js" + ], + "missed": [ + "build/build-validation.js", + "test/issue-4959.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 3607, + "latency_ms": 1125.243, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [ + "lib/validation.js" + ], + "missed": [ + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 3745, + "latency_ms": 1128.434, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js", + "test/close.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3455, + "latency_ms": 1296.456, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1274, + "latency_ms": 532.894, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 439, + "latency_ms": 422.903, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 333, + "latency_ms": 975.247, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts" + ], + "missed": [ + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 711, + "latency_ms": 453.879, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 506, + "latency_ms": 402.058, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 638, + "latency_ms": 439.444, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 580, + "latency_ms": 624.393, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js" + ], + "missed": [ + "test/content-parser.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 412, + "latency_ms": 459.082, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2647, + "latency_ms": 419.113, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1557, + "latency_ms": 430.157, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1406, + "latency_ms": 441.416, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1207, + "latency_ms": 533.395, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "test/trust-proxy.test.js" + ], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 769, + "latency_ms": 446.73, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/decorate.js" + ], + "missed": [ + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 465, + "latency_ms": 454.761, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [], + "missed": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 924, + "latency_ms": 453.683, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1234, + "latency_ms": 447.367, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 536, + "latency_ms": 431.899, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 465, + "latency_ms": 437.093, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3713, + "latency_ms": 420.499, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 960, + "latency_ms": 444.383, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 710, + "latency_ms": 462.5, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 766, + "latency_ms": 435.315, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [ + "test/http2/closing.test.js" + ], + "missed": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.25, + "recall_at_10": 0.25, + "precision_at_10": 0.25, + "recall_at_20": 0.25, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 708, + "latency_ms": 434.409, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 643, + "latency_ms": 439.175, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1576, + "latency_ms": 444.111, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 372, + "latency_ms": 434.754, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 666, + "latency_ms": 434.006, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/errors.js", + "test/web-api.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 676, + "latency_ms": 409.506, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 387, + "latency_ms": 460.431, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 562, + "latency_ms": 721.972, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3157, + "latency_ms": 437.221, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 524, + "latency_ms": 456.414, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 552, + "latency_ms": 431.865, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 782, + "latency_ms": 776.111, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [ + "test/close-pipelining.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 377, + "latency_ms": 425.55, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js", + "test/find-route.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 458, + "latency_ms": 511.409, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 815, + "latency_ms": 443.967, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "test/trust-proxy.test.js" + ], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 822, + "latency_ms": 452.36, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 925, + "latency_ms": 440.945, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/route-prefix.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 795, + "latency_ms": 473.098, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2368, + "latency_ms": 483.289, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 502, + "latency_ms": 457.125, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2163, + "latency_ms": 411.602, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/404s.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 601, + "latency_ms": 445.118, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3131, + "latency_ms": 450.309, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [ + "types/schema.d.ts" + ], + "missed": [ + "test/types/schema.test-d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 807, + "latency_ms": 441.741, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [ + "types/logger.d.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 408, + "latency_ms": 437.98, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 287, + "latency_ms": 841.725, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 812, + "latency_ms": 451.956, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/internals/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 930, + "latency_ms": 415.313, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1273, + "latency_ms": 471.622, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 460, + "latency_ms": 457.432, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 736, + "latency_ms": 462.736, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 586, + "latency_ms": 435.529, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "test/close.test.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 438, + "latency_ms": 998.796, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [ + "test/types/request.test-d.ts" + ], + "missed": [ + "types/request.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1094, + "latency_ms": 624.09, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 464, + "latency_ms": 1383.205, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 420, + "latency_ms": 1001.04, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1473, + "latency_ms": 510.744, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 370, + "latency_ms": 495.996, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2526, + "latency_ms": 883.774, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 545, + "latency_ms": 478.042, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 612, + "latency_ms": 488.044, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 877, + "latency_ms": 476.383, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 447, + "latency_ms": 493.072, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1156, + "latency_ms": 494.693, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 623, + "latency_ms": 557.21, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [ + "types/route.d.ts" + ], + "missed": [ + "test/types/route.test-d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 827, + "latency_ms": 562.779, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/plugin-utils.js" + ], + "missed": [ + "lib/errors.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.5, + "recall_at_10": 0.2, + "precision_at_10": 0.5, + "recall_at_20": 0.2, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 1299, + "latency_ms": 499.881, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 774, + "latency_ms": 568.333, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [ + "lib/validation.js" + ], + "missed": [ + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 993, + "latency_ms": 443.605, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 650, + "latency_ms": 463.855, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 568, + "latency_ms": 454.735, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [ + "lib/schema-controller.js" + ], + "missed": [ + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 641, + "latency_ms": 454.362, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/throw.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1271, + "latency_ms": 443.312, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 823, + "latency_ms": 439.503, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 489, + "latency_ms": 440.653, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 694, + "latency_ms": 434.096, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1345, + "latency_ms": 449.695, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "test/router-options.test.js" + ], + "missed": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.5, + "recall_at_10": 0.1667, + "precision_at_10": 0.5, + "recall_at_20": 0.1667, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 2, + "tokens_delivered": 1174, + "latency_ms": 429.705, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 853, + "latency_ms": 412.88, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "test/close.test.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 834, + "latency_ms": 463.512, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4732, + "latency_ms": 910.02, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5299, + "latency_ms": 1739.923, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "types/instance.d.ts" + ], + "missed": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 4, + "tokens_delivered": 4639, + "latency_ms": 944.756, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4679, + "latency_ms": 1120.419, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5387, + "latency_ms": 911.257, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4080, + "latency_ms": 936.044, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5072, + "latency_ms": 967.462, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4635, + "latency_ms": 1000.537, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4858, + "latency_ms": 1562.236, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4543, + "latency_ms": 1343.392, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5080, + "latency_ms": 915.954, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/errors.js" + ], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.25, + "recall_at_10": 0.3333, + "precision_at_10": 0.25, + "recall_at_20": 0.3333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 5524, + "latency_ms": 1946.639, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5001, + "latency_ms": 941.999, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5152, + "latency_ms": 929.004, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [], + "missed": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4467, + "latency_ms": 937.157, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5095, + "latency_ms": 1024.062, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4753, + "latency_ms": 969.519, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4972, + "latency_ms": 913.566, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [ + "test/types/type-provider.test-d.ts" + ], + "missed": [ + "types/type-provider.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 5028, + "latency_ms": 912.603, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4963, + "latency_ms": 923.427, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4962, + "latency_ms": 914.42, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4961, + "latency_ms": 1024.351, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [ + "test/http2/closing.test.js" + ], + "missed": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 4677, + "latency_ms": 909.394, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4705, + "latency_ms": 928.206, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5222, + "latency_ms": 934.866, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4223, + "latency_ms": 911.959, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4723, + "latency_ms": 911.757, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4675, + "latency_ms": 922.814, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4411, + "latency_ms": 913.32, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 5011, + "latency_ms": 919.729, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 4966, + "latency_ms": 908.687, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3962, + "latency_ms": 904.695, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5394, + "latency_ms": 947.728, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4561, + "latency_ms": 914.851, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5222, + "latency_ms": 910.775, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/find-route.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4584, + "latency_ms": 913.163, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5064, + "latency_ms": 1209.033, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "test/trust-proxy.test.js" + ], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 4692, + "latency_ms": 994.498, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5056, + "latency_ms": 1080.58, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4503, + "latency_ms": 1439.571, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4448, + "latency_ms": 1293.036, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4664, + "latency_ms": 1141.207, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5074, + "latency_ms": 1278.57, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/404s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4189, + "latency_ms": 997.513, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/route.js", + "test/hooks.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 4843, + "latency_ms": 2235.287, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4724, + "latency_ms": 1277.821, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [ + "types/logger.d.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 3636, + "latency_ms": 1340.61, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3755, + "latency_ms": 923.504, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4870, + "latency_ms": 1003.599, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5470, + "latency_ms": 951.055, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5013, + "latency_ms": 952.27, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5378, + "latency_ms": 976.901, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5198, + "latency_ms": 1891.133, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4851, + "latency_ms": 1186.673, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4835, + "latency_ms": 1149.31, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 4742, + "latency_ms": 1448.147, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5286, + "latency_ms": 1853.711, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/fastify-instance.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 5108, + "latency_ms": 1218.507, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [ + "lib/log-controller.js" + ], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 4456, + "latency_ms": 1180.541, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4945, + "latency_ms": 953.963, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 4599, + "latency_ms": 961.354, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4432, + "latency_ms": 1161.528, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4875, + "latency_ms": 971.391, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [ + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts" + ], + "missed": [ + "test/types/instance.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 3, + "tokens_delivered": 5003, + "latency_ms": 1057.95, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4937, + "latency_ms": 948.951, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "missed": [ + "lib/contentTypeParser.js", + "test/buffer.test.js" + ], + "recall_at_5": 0.6, + "precision_at_5": 0.6, + "recall_at_10": 0.6, + "precision_at_10": 0.4286, + "recall_at_20": 0.6, + "precision_at_20": 0.4286, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 2, + "tokens_delivered": 5432, + "latency_ms": 1119.418, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5136, + "latency_ms": 958.868, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5198, + "latency_ms": 1117.015, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "missed": [], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.625, + "recall_at_20": 1, + "precision_at_20": 0.625, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5194, + "latency_ms": 1169.449, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5316, + "latency_ms": 3501.743, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4988, + "latency_ms": 1617.585, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4916, + "latency_ms": 1013.575, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4580, + "latency_ms": 983.81, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4940, + "latency_ms": 1765.242, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4832, + "latency_ms": 1978.928, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4386, + "latency_ms": 958.749, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4586, + "latency_ms": 1285.332, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4526, + "latency_ms": 994.361, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4783, + "latency_ms": 903.681, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "fastify.d.ts" + ], + "missed": [ + "build/build-validation.js", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.1667, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 1, + "tokens_delivered": 4754, + "latency_ms": 926.356, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4416, + "latency_ms": 928.656, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4632, + "latency_ms": 933.284, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/M-hugo-lex.json b/benchmarks/context-retrieval/results/M-hugo-lex.json new file mode 100644 index 00000000..577f9c16 --- /dev/null +++ b/benchmarks/context-retrieval/results/M-hugo-lex.json @@ -0,0 +1,5897 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "hugo", + "head": "7b5199fdeff7d7682e9f979f468d55bc7b67b934" + }, + "tier": "medium", + "tier_code_files": 932, + "corpus_counts": { + "cases": 29, + "path_leak": 13, + "baseline_blind": 16, + "multi_file": 25 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.1006, + "control": 0, + "reason": null + }, + "needs_raw_payload_check": [ + { + "provider_id": "ripgrep", + "why": "scored exactly zero — read the raw payload before believing it" + }, + { + "provider_id": "zoekt", + "why": "scored exactly zero — read the raw payload before believing it" + }, + { + "provider_id": "ast-grep", + "why": "scored exactly zero — read the raw payload before believing it" + } + ], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "Fix potential content XSS by escaping dangerous URLs in links and images", + "revision": "81a5cdca0788ca39574a17d444c9db29d0b19e27", + "returned": [ + "markup/goldmark/images/images_integration_test.go", + "markup/goldmark/images/transform.go", + "tpl/collections/where_test.go", + "modules/collect_test.go", + "markup/goldmark/goldmark_integration_test.go" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix menu pageRef resolution in multidimensional setups", + "revision": "d98cd4aecf25b9df78d811759ea6135b0c7610f1", + "returned": [ + "common/hugo/version_current.go", + "hugolib/site.go", + "hugolib/hugo_sites.go", + "resources/images/config.go", + ".github/workflows/aiwatchdog.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix some default site redirect woes", + "revision": "66ba63cd5f30ef8807fc524dbc67399376299bb1", + "returned": [ + "common/hugo/version_current.go", + "tpl/tplimpl/templatestore.go", + "tpl/tplimpl/templatestore_integration_test.go", + "hugolib/site.go", + ".github/workflows/aiwatchdog.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix index out of range panic in fileEventsContentPaths", + "revision": "e8577771c3275d16b7d1b31cca5e9fd3ea517416", + "returned": [ + "common/hugo/version_current.go", + "hugolib/site.go", + "hugolib/hugo_sites.go", + "resources/images/config.go", + ".github/workflows/aiwatchdog.yml" + ] + }, + { + "provider_id": "ripgrep", + "query": "Fix cascade draft panic", + "revision": "192e3c453e904cf2e79312d4335a7ee216fe659d", + "returned": [ + "hugolib/content_map_page_assembler.go", + "hugolib/content_map_test.go", + "hugolib/page__meta.go", + "config/allconfig/allconfig.go", + "config/allconfig/configlanguage.go" + ] + }, + { + "provider_id": "ripgrep", + "query": "Fix image DecodeConfig regression of WebP images from file cache", + "revision": "6ef8017f60117ad9d900cc59f10a962cd68566d6", + "returned": [ + "config/commonConfig.go", + "livereload/livereload.js", + "livereload/livereload.min.js", + "resources/image.go", + "resources/transform.go" + ] + }, + { + "provider_id": "ripgrep", + "query": "Fix index out of range panic in fileEventsContentPaths", + "revision": "e8577771c3275d16b7d1b31cca5e9fd3ea517416", + "returned": [ + "commands/server.go", + "hugolib/content_map_page_assembler.go", + "hugolib/hugo_sites_build.go", + "hugolib/page__meta.go", + "hugofs/component_fs.go" + ] + }, + { + "provider_id": "ast-grep", + "query": "Fix menu pageRef resolution in multidimensional setups", + "revision": "d98cd4aecf25b9df78d811759ea6135b0c7610f1", + "returned": [ + "navigation/pagemenus.go", + "navigation/menu.go", + "navigation/menu_cache_test.go", + "hugolib/site.go", + "resources/resource_spec.go" + ] + }, + { + "provider_id": "ast-grep", + "query": "Fix it so we never auto-fallback to page resources in other roles/versions", + "revision": "a17bdbc5face02ed2470895bcbfe67db7749dc73", + "returned": [ + "resources/resource/params.go", + "hugolib/content_map_page_contentnodeshifter.go", + "hugolib/content_map_page_contentnode.go", + "hugolib/content_map.go", + "hugolib/doctree/nodeshiftree_test.go" + ] + }, + { + "provider_id": "filename-match", + "query": "Fix partial decorator detection in partial with blocks with outer range break or continue", + "revision": "f42c422a12987c9ccb1927fc17da37feedd0c7fb", + "returned": [ + "tpl/templates/decorator_integration_test.go" + ] + }, + { + "provider_id": "filename-match", + "query": "Fix multi --renderSegments merge behavior", + "revision": "a00b5c72ac57afe26df6688ece3ca544a56df372", + "returned": [ + "hugolib/content_render_hooks_test.go", + "hugolib/pages_language_merge_test.go", + "hugolib/site_render.go", + "hugolib/segments/segments_integration_test.go", + "hugolib/segments/segments_test.go" + ] + }, + { + "provider_id": "filename-match", + "query": "Fix panic when 404 is backed by a content file", + "revision": "5f46da6e2a3f838b40176775c0b46855f4f8f529", + "returned": [ + "create/content_test.go", + "create/content.go", + "helpers/content_test.go", + "helpers/content.go", + "hugolib/content_factory.go" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.0172, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0172, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0345, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0172, + "mean_precision_at_10": 0.0034, + "zero_hit_rate": 0.931, + "median_tokens_delivered": 31301, + "median_latency_ms": 28.221, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.0172, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0172, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0345, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0172, + "mean_precision_at_10": 0.0034, + "zero_hit_rate": 0.931, + "median_tokens_delivered": 31301, + "median_latency_ms": 28.221, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 33227, + "median_latency_ms": 30.487, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.0313, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0313, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0625, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0313, + "mean_precision_at_10": 0.0063, + "zero_hit_rate": 0.875, + "median_tokens_delivered": 29186, + "median_latency_ms": 26.037, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 24754.6, + "peak_rss_mb": 24917.8, + "delta_rss_mb": 163.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.071, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1026, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1938, + "full_recall_at_20": 0.069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0345, + "zero_hit_rate": 0.5862, + "median_tokens_delivered": 145957, + "median_latency_ms": 67.935, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.071, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1026, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1938, + "full_recall_at_20": 0.069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0345, + "zero_hit_rate": 0.5862, + "median_tokens_delivered": 145957, + "median_latency_ms": 67.935, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.0879, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.12, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2335, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0462, + "zero_hit_rate": 0.4615, + "median_tokens_delivered": 141901, + "median_latency_ms": 64.252, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.0573, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0885, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1615, + "full_recall_at_20": 0.0625, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.025, + "zero_hit_rate": 0.6875, + "median_tokens_delivered": 157334.5, + "median_latency_ms": 69.024, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 24626.5, + "peak_rss_mb": 24797.4, + "delta_rss_mb": 170.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "ripgrep", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.2541, + "full_recall_at_5": 0.069, + "mean_recall_at_10": 0.3633, + "full_recall_at_10": 0.1379, + "mean_recall_at_20": 0.5213, + "full_recall_at_20": 0.2069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0998, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.1724, + "median_tokens_delivered": 122785, + "median_latency_ms": 422.645, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.2541, + "full_recall_at_5": 0.069, + "mean_recall_at_10": 0.3633, + "full_recall_at_10": 0.1379, + "mean_recall_at_20": 0.5213, + "full_recall_at_20": 0.2069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0998, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.1724, + "median_tokens_delivered": 122785, + "median_latency_ms": 422.645, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.2592, + "full_recall_at_5": 0.0769, + "mean_recall_at_10": 0.413, + "full_recall_at_10": 0.1538, + "mean_recall_at_20": 0.6245, + "full_recall_at_20": 0.2308, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1328, + "mean_precision_at_10": 0.1308, + "zero_hit_rate": 0, + "median_tokens_delivered": 129765, + "median_latency_ms": 439.996, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.25, + "full_recall_at_5": 0.0625, + "mean_recall_at_10": 0.3229, + "full_recall_at_10": 0.125, + "mean_recall_at_20": 0.4375, + "full_recall_at_20": 0.1875, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0729, + "mean_precision_at_10": 0.075, + "zero_hit_rate": 0.3125, + "median_tokens_delivered": 117161, + "median_latency_ms": 405.846, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 24698.2, + "peak_rss_mb": 26429.3, + "delta_rss_mb": 1731.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "zoekt", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 265.348, + "payload_kind": [ + "whole-files" + ], + "unavailable": 29 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 265.348, + "payload_kind": [ + "whole-files" + ], + "unavailable": 29 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 264.899, + "payload_kind": [ + "whole-files" + ], + "unavailable": 13 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 267.8015, + "payload_kind": [ + "whole-files" + ], + "unavailable": 16 + } + }, + "outcomes": { + "did-not-index": 29 + }, + "resources": { + "baseline_rss_mb": 23810.2, + "peak_rss_mb": 25218.4, + "delta_rss_mb": 1408.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "ast-grep", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.0222, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0271, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0616, + "full_recall_at_20": 0.0345, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0222, + "mean_precision_at_10": 0.0103, + "zero_hit_rate": 0.8966, + "median_tokens_delivered": 51050, + "median_latency_ms": 799.159, + "payload_kind": [ + "whole-files" + ], + "unavailable": 1 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.0222, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0271, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0616, + "full_recall_at_20": 0.0345, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0222, + "mean_precision_at_10": 0.0103, + "zero_hit_rate": 0.8966, + "median_tokens_delivered": 51050, + "median_latency_ms": 799.159, + "payload_kind": [ + "whole-files" + ], + "unavailable": 1 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.0495, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0604, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1374, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0495, + "mean_precision_at_10": 0.0231, + "zero_hit_rate": 0.7692, + "median_tokens_delivered": 53397, + "median_latency_ms": 792.053, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 47682.5, + "median_latency_ms": 823.7355, + "payload_kind": [ + "whole-files" + ], + "unavailable": 1 + } + }, + "outcomes": { + "answered": 27, + "indexed-but-returned-nothing": 2 + }, + "resources": { + "baseline_rss_mb": 22466, + "peak_rss_mb": 22555.8, + "delta_rss_mb": 89.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "filename-match", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.1351, + "full_recall_at_5": 0.0345, + "mean_recall_at_10": 0.181, + "full_recall_at_10": 0.0345, + "mean_recall_at_20": 0.1897, + "full_recall_at_20": 0.0345, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1006, + "mean_recall_at_16000_tokens": 0.1638, + "mean_precision_at_10": 0.1103, + "zero_hit_rate": 0.5862, + "median_tokens_delivered": 27726, + "median_latency_ms": 13.619, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.1351, + "full_recall_at_5": 0.0345, + "mean_recall_at_10": 0.181, + "full_recall_at_10": 0.0345, + "mean_recall_at_20": 0.1897, + "full_recall_at_20": 0.0345, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1006, + "mean_recall_at_16000_tokens": 0.1638, + "mean_precision_at_10": 0.1103, + "zero_hit_rate": 0.5862, + "median_tokens_delivered": 27726, + "median_latency_ms": 13.619, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.3013, + "full_recall_at_5": 0.0769, + "mean_recall_at_10": 0.4038, + "full_recall_at_10": 0.0769, + "mean_recall_at_20": 0.4231, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2244, + "mean_recall_at_16000_tokens": 0.3654, + "mean_precision_at_10": 0.2462, + "zero_hit_rate": 0.0769, + "median_tokens_delivered": 27893, + "median_latency_ms": 13.791, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 24109, + "median_latency_ms": 13.5065, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 26, + "indexed-but-returned-nothing": 3 + }, + "resources": { + "baseline_rss_mb": 20236.7, + "peak_rss_mb": 20242.6, + "delta_rss_mb": 5.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23450, + "latency_ms": 95.092, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43237, + "latency_ms": 36.526, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [], + "missed": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15701, + "latency_ms": 35.932, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [], + "missed": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23223, + "latency_ms": 28.221, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [], + "missed": [ + "commands/server.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31301, + "latency_ms": 30.96, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33227, + "latency_ms": 30.487, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [ + "markup/goldmark/goldmark_integration_test.go" + ], + "missed": [ + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 31381, + "latency_ms": 42.958, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26363, + "latency_ms": 38.683, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32262, + "latency_ms": 40.19, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28830, + "latency_ms": 25.674, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18869, + "latency_ms": 26.047, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [ + "hugolib/embedded_templates_test.go" + ], + "missed": [ + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 23902, + "latency_ms": 26.027, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27186, + "latency_ms": 29.539, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33298, + "latency_ms": 40.436, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [], + "missed": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38765, + "latency_ms": 30.4, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23248, + "latency_ms": 37.027, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32875, + "latency_ms": 26.612, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33433, + "latency_ms": 22.923, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39664, + "latency_ms": 24.299, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24224, + "latency_ms": 25.063, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22664, + "latency_ms": 38.108, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27862, + "latency_ms": 33.219, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44378, + "latency_ms": 25.193, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84924, + "latency_ms": 20.584, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43205, + "latency_ms": 21.839, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43171, + "latency_ms": 22.634, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [], + "missed": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29542, + "latency_ms": 23.688, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34959, + "latency_ms": 26.299, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30727, + "latency_ms": 25.564, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 135284, + "latency_ms": 108.288, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 141901, + "latency_ms": 60.87, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [ + "hugolib/cascade_test.go" + ], + "missed": [ + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 147188, + "latency_ms": 64.252, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [], + "missed": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 167064, + "latency_ms": 68.87, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [], + "missed": [ + "commands/server.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 158300, + "latency_ms": 64.218, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/site.go" + ], + "missed": [ + "hugolib/menu_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 160960, + "latency_ms": 69.173, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [], + "missed": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 156369, + "latency_ms": 65.922, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123900, + "latency_ms": 114.032, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145957, + "latency_ms": 58.868, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 167158, + "latency_ms": 67.703, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 168902, + "latency_ms": 75.839, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 143955, + "latency_ms": 67.935, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [ + "hugolib/integrationtest_builder.go" + ], + "missed": [ + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 148660, + "latency_ms": 61.063, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/sitesmatrix/dimensions.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.2857, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 167157, + "latency_ms": 59.297, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "config/allconfig/allconfig.go", + "hugolib/site.go" + ], + "missed": [ + ".prettierignore", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 140039, + "latency_ms": 66.55, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 158300, + "latency_ms": 74.145, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145753, + "latency_ms": 62.226, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 133497, + "latency_ms": 105.884, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 141218, + "latency_ms": 69.178, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [ + "hugolib/rebuild_test.go" + ], + "missed": [ + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 167064, + "latency_ms": 77.986, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145952, + "latency_ms": 77.975, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159604, + "latency_ms": 63.324, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [ + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 133592, + "latency_ms": 78.42, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [ + "hugolib/rebuild_test.go" + ], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 135377, + "latency_ms": 105.127, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 139252, + "latency_ms": 80.53, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 133591, + "latency_ms": 61.099, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [ + "tpl/css/build_integration_test.go" + ], + "missed": [ + "resources/transform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 158549, + "latency_ms": 57.478, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 163147, + "latency_ms": 65.968, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 138542, + "latency_ms": 81.184, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [ + "tpl/tplimpl/templatetransform.go" + ], + "missed": [ + "tpl/templates/decorator_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 199275, + "latency_ms": 818.28, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 129765, + "latency_ms": 734.222, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "hugolib/cascade_test.go", + "magefile.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 128841, + "latency_ms": 568.135, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 95845, + "latency_ms": 692.425, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 142606, + "latency_ms": 883.542, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 85451, + "latency_ms": 537.834, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [ + "markup/goldmark/render_hooks.go" + ], + "missed": [ + "markup/goldmark/goldmark_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 123325, + "latency_ms": 722.787, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go" + ], + "missed": [ + "helpers/path_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 172712, + "latency_ms": 728.366, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [ + "config/commonConfig.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 112219, + "latency_ms": 414.522, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 98451, + "latency_ms": 416.58, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 155306, + "latency_ms": 571.583, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 124779, + "latency_ms": 379.233, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [ + "hugolib/segments/segments_integration_test.go" + ], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 254429, + "latency_ms": 403.51, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/sitesmatrix/dimensions.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0.2857, + "precision_at_5": 0.4, + "recall_at_10": 0.2857, + "precision_at_10": 0.2, + "recall_at_20": 0.2857, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 2, + "tokens_delivered": 115860, + "latency_ms": 427.745, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + ".prettierignore", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 98117, + "latency_ms": 428.442, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 122103, + "latency_ms": 379.893, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [ + "resources/image.go", + "resources/image_cache.go" + ], + "missed": [ + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 107986, + "latency_ms": 424.716, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 87992, + "latency_ms": 341.45, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 86071, + "latency_ms": 405.865, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [ + "tpl/tplimpl/templatestore.go" + ], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 148886, + "latency_ms": 491.308, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41091, + "latency_ms": 362.441, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 143807, + "latency_ms": 400.459, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [ + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 122785, + "latency_ms": 405.827, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [ + "commands/server.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "missed": [ + "common/paths/path_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 130099, + "latency_ms": 378.882, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go" + ], + "missed": [ + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 226391, + "latency_ms": 439.996, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "hugolib/404_test.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 88561, + "latency_ms": 355.557, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [ + "resources/transform.go" + ], + "missed": [ + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71997, + "latency_ms": 393.123, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/integrationtest_builder.go", + "hugolib/site.go" + ], + "missed": [ + "hugolib/rebuild_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 124353, + "latency_ms": 422.645, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 97547, + "latency_ms": 390.098, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 264.899, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 248.695, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [], + "missed": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 259.71, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [], + "missed": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 256.011, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [], + "missed": [ + "commands/server.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 265.098, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 277.563, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [], + "missed": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 398.786, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 360.714, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 251.503, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 268.99, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 265.348, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 248.062, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 273.858, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 244.805, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [], + "missed": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 249.703, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 274.899, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 273.11, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 253.068, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 263.825, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 285.528, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 277.447, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 266.613, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 249.145, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 257.992, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 259.07, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 528.641, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [], + "missed": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 638.585, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 329.112, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 307.262, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62353, + "latency_ms": 1962.497, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 75778, + "latency_ms": 915.183, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [], + "missed": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 52435, + "latency_ms": 566.009, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [], + "missed": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44451, + "latency_ms": 831.988, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 79579, + "latency_ms": 1122.657, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/site.go" + ], + "missed": [ + "hugolib/menu_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 70729, + "latency_ms": 703.532, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [], + "missed": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51050, + "latency_ms": 941.277, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45737, + "latency_ms": 760.878, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38642, + "latency_ms": 700.67, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34880, + "latency_ms": 693.192, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38040, + "latency_ms": 1044.906, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 638.355, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35034, + "latency_ms": 735.785, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [ + "hugolib/content_map.go", + "hugolib/content_map_page.go" + ], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.2857, + "precision_at_10": 0.2, + "recall_at_20": 0.2857, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 4, + "tokens_delivered": 53397, + "latency_ms": 792.053, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [], + "missed": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41420, + "latency_ms": 621.675, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 50914, + "latency_ms": 817.002, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38336, + "latency_ms": 962.607, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 506.006, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 75722, + "latency_ms": 830.469, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117994, + "latency_ms": 991.097, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30979, + "latency_ms": 519.131, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67067, + "latency_ms": 837.918, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 57464, + "latency_ms": 839.033, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63525, + "latency_ms": 799.159, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35373, + "latency_ms": 997.287, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64387, + "latency_ms": 638.041, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [], + "missed": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64476, + "latency_ms": 743.306, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51885, + "latency_ms": 881.152, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43337, + "latency_ms": 694.539, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [ + "tpl/templates/decorator_integration_test.go" + ], + "missed": [ + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2263, + "latency_ms": 15.143, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/hugo_sites.go" + ], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 42078, + "latency_ms": 14.15, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [ + "hugolib/cascade_test.go" + ], + "missed": [ + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 2286, + "latency_ms": 13.791, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [], + "missed": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22749, + "latency_ms": 14.051, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 16766, + "latency_ms": 13.271, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/menu_test.go" + ], + "missed": [ + "hugolib/site.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 52415, + "latency_ms": 13.05, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [], + "missed": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54331, + "latency_ms": 13.821, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [ + "commands/commandeer.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 62484, + "latency_ms": 13.619, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23089, + "latency_ms": 13.52, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7035, + "latency_ms": 12.752, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31794, + "latency_ms": 14.314, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 8.12, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [ + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "missed": [ + "hugolib/integrationtest_builder.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 27893, + "latency_ms": 14.341, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24783, + "latency_ms": 13.873, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "hugolib/site.go", + "hugolib/site_render.go" + ], + "missed": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 3, + "tokens_delivered": 26758, + "latency_ms": 14.015, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17021, + "latency_ms": 13.917, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [ + "resources/image_cache.go" + ], + "missed": [ + "resources/image.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 27726, + "latency_ms": 14.133, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 7.693, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7644, + "latency_ms": 13.493, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32602, + "latency_ms": 13.456, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 7.237, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25129, + "latency_ms": 13.264, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 59510, + "latency_ms": 13.016, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [ + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 62439, + "latency_ms": 12.884, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go" + ], + "missed": [ + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 31534, + "latency_ms": 13.514, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "hugolib/404_test.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 54666, + "latency_ms": 12.899, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [], + "missed": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 66552, + "latency_ms": 13.752, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 58276, + "latency_ms": 15.234, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35377, + "latency_ms": 14.767, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/M-hugo-ret.json b/benchmarks/context-retrieval/results/M-hugo-ret.json new file mode 100644 index 00000000..8fc3fe7b --- /dev/null +++ b/benchmarks/context-retrieval/results/M-hugo-ret.json @@ -0,0 +1,4922 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "hugo", + "head": "7b5199fdeff7d7682e9f979f468d55bc7b67b934" + }, + "tier": "medium", + "tier_code_files": 932, + "corpus_counts": { + "cases": 29, + "path_leak": 13, + "baseline_blind": 16, + "multi_file": 25 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.4503, + "control": 0, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "Fix potential content XSS by escaping dangerous URLs in links and images", + "revision": "81a5cdca0788ca39574a17d444c9db29d0b19e27", + "returned": [ + "markup/goldmark/images/images_integration_test.go", + "markup/goldmark/images/transform.go", + "tpl/collections/where_test.go", + "modules/collect_test.go", + "markup/goldmark/goldmark_integration_test.go" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix menu pageRef resolution in multidimensional setups", + "revision": "d98cd4aecf25b9df78d811759ea6135b0c7610f1", + "returned": [ + "common/hugo/version_current.go", + "hugolib/site.go", + "hugolib/hugo_sites.go", + "resources/images/config.go", + ".github/workflows/aiwatchdog.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix some default site redirect woes", + "revision": "66ba63cd5f30ef8807fc524dbc67399376299bb1", + "returned": [ + "common/hugo/version_current.go", + "tpl/tplimpl/templatestore.go", + "tpl/tplimpl/templatestore_integration_test.go", + "hugolib/site.go", + ".github/workflows/aiwatchdog.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix index out of range panic in fileEventsContentPaths", + "revision": "e8577771c3275d16b7d1b31cca5e9fd3ea517416", + "returned": [ + "common/hugo/version_current.go", + "hugolib/site.go", + "hugolib/hugo_sites.go", + "resources/images/config.go", + ".github/workflows/aiwatchdog.yml" + ] + }, + { + "provider_id": "semble", + "query": "Fix partial decorator detection in partial with blocks with outer range break or continue", + "revision": "f42c422a12987c9ccb1927fc17da37feedd0c7fb", + "returned": [ + "tpl/tplimpl/templatetransform.go", + "tpl/templates/templates.go", + "tpl/partials/partials.go", + "tpl/tplimpl/templatestore.go", + "hugolib/integrationtest_builder.go" + ] + }, + { + "provider_id": "semble", + "query": "Fix some default site redirect woes", + "revision": "66ba63cd5f30ef8807fc524dbc67399376299bb1", + "returned": [ + "hugolib/site_render.go", + "resources/page/site.go", + "commands/server.go", + "config/commonConfig.go", + "hugolib/site.go" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix partial decorator detection in partial with blocks with outer range break or continue", + "revision": "f42c422a12987c9ccb1927fc17da37feedd0c7fb", + "returned": [ + "tpl/templates/decorator_integration_test.go" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix multi --renderSegments merge behavior", + "revision": "a00b5c72ac57afe26df6688ece3ca544a56df372", + "returned": [ + "docs/data/docs.yaml", + "hugolib/segments/segments_integration_test.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_test.go" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix index out of range panic in fileEventsContentPaths", + "revision": "e8577771c3275d16b7d1b31cca5e9fd3ea517416", + "returned": [ + "hugolib/site.go", + "commands/server.go", + "watcher/filenotify/poller_test.go" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "Fix RenderShortcodes leaking context markers when indented", + "revision": "45e4596630b1372ab02634e4536d2b5a89452e0b", + "returned": [ + "docs/content/en/showcase/_index.md", + "docs/content/en/functions/transform/HighlightCodeBlock.md", + "docs/content/en/quick-reference/glossary/layout.md", + "docs/content/en/methods/page/RenderShortcodes.md", + "docs/assets/css/components/shortcodes.css" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.0172, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0172, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0345, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0172, + "mean_precision_at_10": 0.0034, + "zero_hit_rate": 0.931, + "median_tokens_delivered": 31301, + "median_latency_ms": 13.627, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.0172, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0172, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0345, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0172, + "mean_precision_at_10": 0.0034, + "zero_hit_rate": 0.931, + "median_tokens_delivered": 31301, + "median_latency_ms": 13.627, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 33227, + "median_latency_ms": 13.26, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.0313, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0313, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0625, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0313, + "mean_precision_at_10": 0.0063, + "zero_hit_rate": 0.875, + "median_tokens_delivered": 29186, + "median_latency_ms": 13.7425, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 20555.5, + "peak_rss_mb": 20557.5, + "delta_rss_mb": 2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.071, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1026, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1938, + "full_recall_at_20": 0.069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0345, + "zero_hit_rate": 0.5862, + "median_tokens_delivered": 145957, + "median_latency_ms": 34.509, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.071, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1026, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1938, + "full_recall_at_20": 0.069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0345, + "zero_hit_rate": 0.5862, + "median_tokens_delivered": 145957, + "median_latency_ms": 34.509, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.0879, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.12, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2335, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0462, + "zero_hit_rate": 0.4615, + "median_tokens_delivered": 141901, + "median_latency_ms": 35.48, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.0573, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0885, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1615, + "full_recall_at_20": 0.0625, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.025, + "zero_hit_rate": 0.6875, + "median_tokens_delivered": 157334.5, + "median_latency_ms": 33.72, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 20574.8, + "peak_rss_mb": 20616.9, + "delta_rss_mb": 42.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.2759, + "full_recall_at_5": 0.069, + "mean_recall_at_10": 0.3276, + "full_recall_at_10": 0.069, + "mean_recall_at_20": 0.4676, + "full_recall_at_20": 0.1724, + "mean_recall_at_1000_tokens": 0.2615, + "mean_recall_at_4000_tokens": 0.4503, + "mean_recall_at_16000_tokens": 0.4676, + "mean_precision_at_10": 0.0793, + "zero_hit_rate": 0.1379, + "median_tokens_delivered": 3945, + "median_latency_ms": 3516.448, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.2759, + "full_recall_at_5": 0.069, + "mean_recall_at_10": 0.3276, + "full_recall_at_10": 0.069, + "mean_recall_at_20": 0.4676, + "full_recall_at_20": 0.1724, + "mean_recall_at_1000_tokens": 0.2615, + "mean_recall_at_4000_tokens": 0.4503, + "mean_recall_at_16000_tokens": 0.4676, + "mean_precision_at_10": 0.0793, + "zero_hit_rate": 0.1379, + "median_tokens_delivered": 3945, + "median_latency_ms": 3516.448, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.2628, + "full_recall_at_5": 0.0769, + "mean_recall_at_10": 0.3397, + "full_recall_at_10": 0.0769, + "mean_recall_at_20": 0.4789, + "full_recall_at_20": 0.1538, + "mean_recall_at_1000_tokens": 0.2308, + "mean_recall_at_4000_tokens": 0.4789, + "mean_recall_at_16000_tokens": 0.4789, + "mean_precision_at_10": 0.0923, + "zero_hit_rate": 0, + "median_tokens_delivered": 3976, + "median_latency_ms": 3519.739, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.2865, + "full_recall_at_5": 0.0625, + "mean_recall_at_10": 0.3177, + "full_recall_at_10": 0.0625, + "mean_recall_at_20": 0.4583, + "full_recall_at_20": 0.1875, + "mean_recall_at_1000_tokens": 0.2865, + "mean_recall_at_4000_tokens": 0.4271, + "mean_recall_at_16000_tokens": 0.4583, + "mean_precision_at_10": 0.0687, + "zero_hit_rate": 0.25, + "median_tokens_delivered": 3909, + "median_latency_ms": 3513.5625, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 20595.1, + "peak_rss_mb": 23870.3, + "delta_rss_mb": 3275.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "jcodemunch", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.1839, + "full_recall_at_5": 0.0345, + "mean_recall_at_10": 0.1839, + "full_recall_at_10": 0.0345, + "mean_recall_at_20": 0.1839, + "full_recall_at_20": 0.0345, + "mean_recall_at_1000_tokens": 0.1839, + "mean_recall_at_4000_tokens": 0.1839, + "mean_recall_at_16000_tokens": 0.1839, + "mean_precision_at_10": 0.2529, + "zero_hit_rate": 0.5862, + "median_tokens_delivered": 565, + "median_latency_ms": 916.254, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 3 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.1839, + "full_recall_at_5": 0.0345, + "mean_recall_at_10": 0.1839, + "full_recall_at_10": 0.0345, + "mean_recall_at_20": 0.1839, + "full_recall_at_20": 0.0345, + "mean_recall_at_1000_tokens": 0.1839, + "mean_recall_at_4000_tokens": 0.1839, + "mean_recall_at_16000_tokens": 0.1839, + "mean_precision_at_10": 0.2529, + "zero_hit_rate": 0.5862, + "median_tokens_delivered": 565, + "median_latency_ms": 916.254, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 3 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.2821, + "full_recall_at_5": 0.0769, + "mean_recall_at_10": 0.2821, + "full_recall_at_10": 0.0769, + "mean_recall_at_20": 0.2821, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0.2821, + "mean_recall_at_4000_tokens": 0.2821, + "mean_recall_at_16000_tokens": 0.2821, + "mean_precision_at_10": 0.4359, + "zero_hit_rate": 0.3846, + "median_tokens_delivered": 565, + "median_latency_ms": 912.22, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.1042, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1042, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1042, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.1042, + "mean_recall_at_4000_tokens": 0.1042, + "mean_recall_at_16000_tokens": 0.1042, + "mean_precision_at_10": 0.1042, + "zero_hit_rate": 0.75, + "median_tokens_delivered": 566, + "median_latency_ms": 932.09, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 3 + } + }, + "outcomes": { + "answered": 26, + "indexed-but-returned-nothing": 3 + }, + "resources": { + "baseline_rss_mb": 20640.8, + "peak_rss_mb": 23825.2, + "delta_rss_mb": 3184.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "cocoindex-code", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.0345, + "full_recall_at_5": 0.0345, + "mean_recall_at_10": 0.1437, + "full_recall_at_10": 0.069, + "mean_recall_at_20": 0.1839, + "full_recall_at_20": 0.069, + "mean_recall_at_1000_tokens": 0.0057, + "mean_recall_at_4000_tokens": 0.1667, + "mean_recall_at_16000_tokens": 0.1839, + "mean_precision_at_10": 0.0276, + "zero_hit_rate": 0.6552, + "median_tokens_delivered": 3742, + "median_latency_ms": 2652.89, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.0345, + "full_recall_at_5": 0.0345, + "mean_recall_at_10": 0.1437, + "full_recall_at_10": 0.069, + "mean_recall_at_20": 0.1839, + "full_recall_at_20": 0.069, + "mean_recall_at_1000_tokens": 0.0057, + "mean_recall_at_4000_tokens": 0.1667, + "mean_recall_at_16000_tokens": 0.1839, + "mean_precision_at_10": 0.0276, + "zero_hit_rate": 0.6552, + "median_tokens_delivered": 3742, + "median_latency_ms": 2652.89, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0897, + "full_recall_at_10": 0.0769, + "mean_recall_at_20": 0.1795, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0.0128, + "mean_recall_at_4000_tokens": 0.141, + "mean_recall_at_16000_tokens": 0.1795, + "mean_precision_at_10": 0.0154, + "zero_hit_rate": 0.6923, + "median_tokens_delivered": 3742, + "median_latency_ms": 2518.959, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.0625, + "full_recall_at_5": 0.0625, + "mean_recall_at_10": 0.1875, + "full_recall_at_10": 0.0625, + "mean_recall_at_20": 0.1875, + "full_recall_at_20": 0.0625, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1875, + "mean_recall_at_16000_tokens": 0.1875, + "mean_precision_at_10": 0.0375, + "zero_hit_rate": 0.625, + "median_tokens_delivered": 3744, + "median_latency_ms": 2665.06, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 23586.4, + "peak_rss_mb": 39028.5, + "delta_rss_mb": 15442.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23450, + "latency_ms": 20.213, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43237, + "latency_ms": 14.067, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [], + "missed": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15701, + "latency_ms": 14.904, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [], + "missed": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23223, + "latency_ms": 13.834, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [], + "missed": [ + "commands/server.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31301, + "latency_ms": 13.667, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33227, + "latency_ms": 13.191, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [ + "markup/goldmark/goldmark_integration_test.go" + ], + "missed": [ + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 31381, + "latency_ms": 12.581, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26363, + "latency_ms": 12.932, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32262, + "latency_ms": 14.065, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28830, + "latency_ms": 13.96, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18869, + "latency_ms": 14.338, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [ + "hugolib/embedded_templates_test.go" + ], + "missed": [ + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 23902, + "latency_ms": 14.132, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27186, + "latency_ms": 13.036, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33298, + "latency_ms": 12.921, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [], + "missed": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38765, + "latency_ms": 12.178, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23248, + "latency_ms": 12.052, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32875, + "latency_ms": 12.546, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33433, + "latency_ms": 12.185, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39664, + "latency_ms": 13.245, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24224, + "latency_ms": 13.651, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22664, + "latency_ms": 14.923, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27862, + "latency_ms": 13.448, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44378, + "latency_ms": 13.936, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84924, + "latency_ms": 14.677, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43205, + "latency_ms": 13.26, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43171, + "latency_ms": 13.627, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [], + "missed": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29542, + "latency_ms": 14.181, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34959, + "latency_ms": 12.543, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30727, + "latency_ms": 12.887, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 135284, + "latency_ms": 39.562, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 141901, + "latency_ms": 32.943, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [ + "hugolib/cascade_test.go" + ], + "missed": [ + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 147188, + "latency_ms": 35.606, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [], + "missed": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 167064, + "latency_ms": 34.837, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [], + "missed": [ + "commands/server.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 158300, + "latency_ms": 34.383, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/site.go" + ], + "missed": [ + "hugolib/menu_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 160960, + "latency_ms": 34.07, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [], + "missed": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 156369, + "latency_ms": 33.823, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123900, + "latency_ms": 54.025, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145957, + "latency_ms": 31.731, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 167158, + "latency_ms": 33.617, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 168902, + "latency_ms": 35.778, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 143955, + "latency_ms": 33.594, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [ + "hugolib/integrationtest_builder.go" + ], + "missed": [ + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 148660, + "latency_ms": 31.617, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/sitesmatrix/dimensions.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.2857, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 167157, + "latency_ms": 34.509, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "config/allconfig/allconfig.go", + "hugolib/site.go" + ], + "missed": [ + ".prettierignore", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 140039, + "latency_ms": 37.189, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 158300, + "latency_ms": 36.22, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145753, + "latency_ms": 33.551, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 133497, + "latency_ms": 50.612, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 141218, + "latency_ms": 36.485, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [ + "hugolib/rebuild_test.go" + ], + "missed": [ + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 167064, + "latency_ms": 35.129, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145952, + "latency_ms": 31.94, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159604, + "latency_ms": 33.067, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [ + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 133592, + "latency_ms": 38.686, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [ + "hugolib/rebuild_test.go" + ], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 135377, + "latency_ms": 53.097, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 139252, + "latency_ms": 36.202, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 133591, + "latency_ms": 35.48, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [ + "tpl/css/build_integration_test.go" + ], + "missed": [ + "resources/transform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 158549, + "latency_ms": 30.711, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 163147, + "latency_ms": 32.333, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 138542, + "latency_ms": 33.454, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3824, + "latency_ms": 5188.25, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/hugo_sites.go" + ], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 4011, + "latency_ms": 3448.728, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "hugolib/cascade_test.go", + "magefile.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 3747, + "latency_ms": 3421.384, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3945, + "latency_ms": 3510.677, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3976, + "latency_ms": 3519.739, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/site.go" + ], + "missed": [ + "hugolib/menu_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 4362, + "latency_ms": 3527.413, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [], + "missed": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4122, + "latency_ms": 3724.742, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [ + "commands/commandeer.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 3720, + "latency_ms": 3330.935, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [ + "config/commonConfig.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3795, + "latency_ms": 4026.043, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [ + "markup/goldmark/toc.go" + ], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 14, + "tokens_delivered": 4280, + "latency_ms": 3728.013, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [ + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3814, + "latency_ms": 3516.448, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [ + "tpl/templates/templates.go" + ], + "missed": [ + "hugolib/embedded_templates_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 4002, + "latency_ms": 3370.786, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [ + "hugolib/segments/segments.go" + ], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 6, + "tokens_delivered": 3985, + "latency_ms": 3618.548, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [ + "hugolib/content_map_page.go" + ], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.1429, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1429, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 12, + "tokens_delivered": 4140, + "latency_ms": 3622.106, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + ".prettierignore", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.6667, + "precision_at_20": 0.2353, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3831, + "latency_ms": 3453.912, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 4287, + "latency_ms": 3599.271, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [ + "resources/image.go", + "resources/image_cache.go" + ], + "missed": [ + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 11, + "tokens_delivered": 3659, + "latency_ms": 3498.662, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3816, + "latency_ms": 3383.395, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugofs/component_fs.go" + ], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 3873, + "latency_ms": 3409.921, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [ + "tpl/tplimpl/templatestore.go" + ], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 4397, + "latency_ms": 3502.102, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3604, + "latency_ms": 3462.56, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 4176, + "latency_ms": 3504.737, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [ + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 3812, + "latency_ms": 3398.919, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [ + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 4179, + "latency_ms": 3398.768, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [ + "internal/warpc/webp.go" + ], + "missed": [ + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 5, + "tokens_delivered": 3731, + "latency_ms": 4155.543, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "hugolib/404_test.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 13, + "tokens_delivered": 4025, + "latency_ms": 3534.806, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [ + "resources/transform.go" + ], + "missed": [ + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3823, + "latency_ms": 3639.774, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/integrationtest_builder.go", + "hugolib/site.go" + ], + "missed": [ + "hugolib/rebuild_test.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 3817, + "latency_ms": 3529.186, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4080, + "latency_ms": 3567.929, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [ + "tpl/templates/decorator_integration_test.go" + ], + "missed": [ + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 561, + "latency_ms": 928.691, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugolib/hugo_sites.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1191, + "latency_ms": 898.841, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [], + "missed": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 412, + "latency_ms": 896.525, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [ + "hugolib/rendershortcodes_test.go" + ], + "missed": [ + "hugolib/page__content.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 531, + "latency_ms": 905.879, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 717, + "latency_ms": 955.864, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 408, + "latency_ms": 903.483, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [ + "markup/goldmark/render_hooks.go" + ], + "missed": [ + "markup/goldmark/goldmark_integration_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 478, + "latency_ms": 912.738, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [ + "commands/commandeer.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.3333, + "recall_at_10": 0.25, + "precision_at_10": 0.3333, + "recall_at_20": 0.25, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 697, + "latency_ms": 880.154, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 425, + "latency_ms": 1394.908, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 573, + "latency_ms": 945.682, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4460, + "latency_ms": 916.254, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 728, + "latency_ms": 891.604, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [ + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "missed": [ + "hugolib/integrationtest_builder.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.5, + "recall_at_10": 0.6667, + "precision_at_10": 0.5, + "recall_at_20": 0.6667, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 484, + "latency_ms": 911.035, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1141, + "latency_ms": 914.633, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0.1667, + "precision_at_5": 1, + "recall_at_10": 0.1667, + "precision_at_10": 1, + "recall_at_20": 0.1667, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 1, + "tokens_delivered": 592, + "latency_ms": 912.22, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 655, + "latency_ms": 981.611, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [ + "resources/image.go" + ], + "missed": [ + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 596, + "latency_ms": 914.645, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 559, + "latency_ms": 897.181, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 970, + "latency_ms": 920.905, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3669, + "latency_ms": 939.825, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 461, + "latency_ms": 908.373, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 410, + "latency_ms": 998.341, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 737, + "latency_ms": 976.29, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [ + "hugolib/rebuild_test.go" + ], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "main_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 1, + "recall_at_10": 0.25, + "precision_at_10": 1, + "recall_at_20": 0.25, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 509, + "latency_ms": 907.226, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 548, + "latency_ms": 976.91, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 565, + "latency_ms": 930.705, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [ + "resources/transform.go" + ], + "missed": [ + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 611, + "latency_ms": 954.343, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/site.go" + ], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 491, + "latency_ms": 933.357, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 530, + "latency_ms": 930.823, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [ + "tpl/tplimpl/templatetransform.go" + ], + "missed": [ + "tpl/templates/decorator_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 4785, + "latency_ms": 4798.9, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/hugo_sites.go" + ], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 3372, + "latency_ms": 2204.29, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [], + "missed": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2712, + "latency_ms": 3401.289, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [ + "hugolib/page__content.go" + ], + "missed": [ + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 10, + "tokens_delivered": 3375, + "latency_ms": 5963.365, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 3742, + "latency_ms": 2518.959, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3129, + "latency_ms": 2018.005, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [], + "missed": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4041, + "latency_ms": 2092.622, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3741, + "latency_ms": 3807.76, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [ + "config/commonConfig.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4086, + "latency_ms": 2476.136, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5095, + "latency_ms": 2476.383, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3135, + "latency_ms": 4298.861, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3924, + "latency_ms": 3205.134, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4999, + "latency_ms": 2182.985, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3643, + "latency_ms": 1863.358, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "hugolib/site.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "missed": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 6, + "tokens_delivered": 2649, + "latency_ms": 1983.897, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3443, + "latency_ms": 1972.115, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4905, + "latency_ms": 2020.259, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3747, + "latency_ms": 2677.23, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 3726, + "latency_ms": 2266.876, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [ + "tpl/tplimpl/templatestore.go" + ], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 8, + "tokens_delivered": 3741, + "latency_ms": 2318.676, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2754, + "latency_ms": 4756.565, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3617, + "latency_ms": 6988.408, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3174, + "latency_ms": 3523.449, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3959, + "latency_ms": 2997.09, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4050, + "latency_ms": 5206.674, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3746, + "latency_ms": 3880.802, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [ + "resources/transform.go" + ], + "missed": [ + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 4402, + "latency_ms": 3084.246, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/site.go" + ], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 6, + "tokens_delivered": 5086, + "latency_ms": 2652.89, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3785, + "latency_ms": 2505.387, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/P-email-manager.json b/benchmarks/context-retrieval/results/P-email-manager.json new file mode 100644 index 00000000..e23153a5 --- /dev/null +++ b/benchmarks/context-retrieval/results/P-email-manager.json @@ -0,0 +1,7012 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "email-manager", + "head": "6750f6ee35bd70bd5e1b31e9e5308b019e2d67d8" + }, + "tier": "small", + "tier_code_files": 102, + "corpus_counts": { + "cases": 47, + "path_leak": 21, + "baseline_blind": 26, + "multi_file": 14 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.3605, + "control": 0.0566, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "hide HF transformers import from server bundler", + "revision": "7796afb6ea63065a43e425264ab2bd72977456b8", + "returned": [ + "src/lib/auth.ts", + "tsconfig.json", + "src/lib/embeddings.ts", + "src/components/Analytics.tsx", + ".github/workflows/weekly.yml" + ] + }, + { + "provider_id": "random-files", + "query": "Fix 500 errors: add token refresh and API error handling", + "revision": "5a0df4b3fee1874f78e91f6fb283bcf3559cb10a", + "returned": [ + "src/app/page.tsx", + "src/components/EmailDetail.tsx", + "src/lib/auth.ts", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "hide HF transformers import from server bundler", + "revision": "7796afb6ea63065a43e425264ab2bd72977456b8", + "returned": [ + "package.json", + "pnpm-lock.yaml", + "src/app/page.tsx", + "next.config.ts", + "src/components/Analytics.tsx" + ] + }, + { + "provider_id": "churn-ranked", + "query": "create better-auth migrations + schema mapping for D1", + "revision": "3cbd63102c967b7ad151f44592049b080c876d6a", + "returned": [ + "package.json", + "pnpm-lock.yaml", + "src/app/page.tsx", + "next.config.ts", + "src/components/Analytics.tsx" + ] + }, + { + "provider_id": "semble", + "query": "migrate deploy from CF Pages to CF Workers, simplify next.config", + "revision": "2c1993e3e71ae79b67cb8b3ca68a62747c88e052", + "returned": [ + "open-next.config.ts", + "next.config.ts", + "eslint.config.js", + "postcss.config.mjs", + "scripts/patch-opennext.mjs" + ] + }, + { + "provider_id": "semble", + "query": "Fix 500 errors: add token refresh and API error handling", + "revision": "5a0df4b3fee1874f78e91f6fb283bcf3559cb10a", + "returned": [ + "src/lib/gmail.ts", + "src/app/api/emails/route.ts", + "src/lib/auth.ts", + "postcss.config.mjs", + "next.config.ts" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix React hooks crash and console errors on /app.", + "revision": "95a35f10d8b779411cbd199d18a9a4543a4382de", + "returned": [ + "src/lib/foundry-monitoring.ts", + "pnpm-lock.yaml" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix Open app serving landing page instead of SPA shell.", + "revision": "e1bbe3e6916311a0ea401a3af73c166721523c54", + "returned": [ + "vite.config.ts", + "src/worker.ts" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "migrate deploy from CF Pages to CF Workers, simplify next.config", + "revision": "2c1993e3e71ae79b67cb8b3ca68a62747c88e052", + "returned": [ + "wrangler.toml", + "foundry.json", + "README.md", + "package.json", + "pnpm-lock.yaml" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "replace broken next lint with tsc, add tsconfig include/paths/relaxed flags", + "revision": "4f284d35f52fc0acddd83977448c5190e0ce2baf", + "returned": [ + "pnpm-lock.yaml", + "tsconfig.json" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 47, + "mean_recall_at_5": 0.08, + "full_recall_at_5": 0.0426, + "mean_recall_at_10": 0.1996, + "full_recall_at_10": 0.1489, + "mean_recall_at_20": 0.3246, + "full_recall_at_20": 0.2128, + "mean_recall_at_1000_tokens": 0.0213, + "mean_recall_at_4000_tokens": 0.0566, + "mean_recall_at_16000_tokens": 0.1748, + "mean_precision_at_10": 0.034, + "zero_hit_rate": 0.5319, + "median_tokens_delivered": 16304, + "median_latency_ms": 14.411, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 40, + "mean_recall_at_5": 0.044, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0596, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2065, + "full_recall_at_20": 0.075, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0165, + "mean_recall_at_16000_tokens": 0.1304, + "mean_precision_at_10": 0.0225, + "zero_hit_rate": 0.625, + "median_tokens_delivered": 16080, + "median_latency_ms": 14.4565, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 7, + "mean_recall_at_5": 0.2857, + "full_recall_at_5": 0.2857, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.1429, + "mean_recall_at_4000_tokens": 0.2857, + "mean_recall_at_16000_tokens": 0.4286, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 76352, + "median_latency_ms": 13.658, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.0694, + "full_recall_at_5": 0.0476, + "mean_recall_at_10": 0.2123, + "full_recall_at_10": 0.1905, + "mean_recall_at_20": 0.3988, + "full_recall_at_20": 0.2857, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0218, + "mean_recall_at_16000_tokens": 0.1806, + "mean_precision_at_10": 0.0286, + "zero_hit_rate": 0.4286, + "median_tokens_delivered": 18158, + "median_latency_ms": 14.203, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 26, + "mean_recall_at_5": 0.0885, + "full_recall_at_5": 0.0385, + "mean_recall_at_10": 0.1894, + "full_recall_at_10": 0.1154, + "mean_recall_at_20": 0.2647, + "full_recall_at_20": 0.1538, + "mean_recall_at_1000_tokens": 0.0385, + "mean_recall_at_4000_tokens": 0.0846, + "mean_recall_at_16000_tokens": 0.1702, + "mean_precision_at_10": 0.0385, + "zero_hit_rate": 0.6154, + "median_tokens_delivered": 14071.5, + "median_latency_ms": 14.4565, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 47 + }, + "resources": { + "baseline_rss_mb": 16829.2, + "peak_rss_mb": 17147.4, + "delta_rss_mb": 318.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 47, + "mean_recall_at_5": 0.2473, + "full_recall_at_5": 0.2128, + "mean_recall_at_10": 0.3727, + "full_recall_at_10": 0.2979, + "mean_recall_at_20": 0.5906, + "full_recall_at_20": 0.5106, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0319, + "mean_recall_at_16000_tokens": 0.2, + "mean_precision_at_10": 0.0553, + "zero_hit_rate": 0.3404, + "median_tokens_delivered": 84238, + "median_latency_ms": 24.07, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 40, + "mean_recall_at_5": 0.2406, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.3379, + "full_recall_at_10": 0.25, + "mean_recall_at_20": 0.569, + "full_recall_at_20": 0.475, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0375, + "mean_recall_at_16000_tokens": 0.185, + "mean_precision_at_10": 0.055, + "zero_hit_rate": 0.35, + "median_tokens_delivered": 88809, + "median_latency_ms": 24.1425, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 7, + "mean_recall_at_5": 0.2857, + "full_recall_at_5": 0.2857, + "mean_recall_at_10": 0.5714, + "full_recall_at_10": 0.5714, + "mean_recall_at_20": 0.7143, + "full_recall_at_20": 0.7143, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.2857, + "mean_precision_at_10": 0.0571, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 81053, + "median_latency_ms": 22.98, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.125, + "full_recall_at_5": 0.0952, + "mean_recall_at_10": 0.2817, + "full_recall_at_10": 0.1905, + "mean_recall_at_20": 0.4861, + "full_recall_at_20": 0.381, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0238, + "mean_recall_at_16000_tokens": 0.119, + "mean_precision_at_10": 0.0524, + "zero_hit_rate": 0.4286, + "median_tokens_delivered": 86151, + "median_latency_ms": 23.834, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 26, + "mean_recall_at_5": 0.3462, + "full_recall_at_5": 0.3077, + "mean_recall_at_10": 0.4462, + "full_recall_at_10": 0.3846, + "mean_recall_at_20": 0.675, + "full_recall_at_20": 0.6154, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0385, + "mean_recall_at_16000_tokens": 0.2654, + "mean_precision_at_10": 0.0577, + "zero_hit_rate": 0.2692, + "median_tokens_delivered": 83901, + "median_latency_ms": 24.222, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 47 + }, + "resources": { + "baseline_rss_mb": 16398.1, + "peak_rss_mb": 16754, + "delta_rss_mb": 355.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 47, + "mean_recall_at_5": 0.3037, + "full_recall_at_5": 0.234, + "mean_recall_at_10": 0.3605, + "full_recall_at_10": 0.2766, + "mean_recall_at_20": 0.3605, + "full_recall_at_20": 0.2766, + "mean_recall_at_1000_tokens": 0.1551, + "mean_recall_at_4000_tokens": 0.3605, + "mean_recall_at_16000_tokens": 0.3605, + "mean_precision_at_10": 0.0775, + "zero_hit_rate": 0.5106, + "median_tokens_delivered": 3448, + "median_latency_ms": 587.92, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 40, + "mean_recall_at_5": 0.2819, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.3485, + "full_recall_at_10": 0.25, + "mean_recall_at_20": 0.3485, + "full_recall_at_20": 0.25, + "mean_recall_at_1000_tokens": 0.1323, + "mean_recall_at_4000_tokens": 0.3485, + "mean_recall_at_16000_tokens": 0.3485, + "mean_precision_at_10": 0.0771, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 3491.5, + "median_latency_ms": 631.8595, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 7, + "mean_recall_at_5": 0.4286, + "full_recall_at_5": 0.4286, + "mean_recall_at_10": 0.4286, + "full_recall_at_10": 0.4286, + "mean_recall_at_20": 0.4286, + "full_recall_at_20": 0.4286, + "mean_recall_at_1000_tokens": 0.2857, + "mean_recall_at_4000_tokens": 0.4286, + "mean_recall_at_16000_tokens": 0.4286, + "mean_precision_at_10": 0.0798, + "zero_hit_rate": 0.5714, + "median_tokens_delivered": 3220, + "median_latency_ms": 543.618, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.4008, + "full_recall_at_5": 0.3333, + "mean_recall_at_10": 0.4167, + "full_recall_at_10": 0.3333, + "mean_recall_at_20": 0.4167, + "full_recall_at_20": 0.3333, + "mean_recall_at_1000_tokens": 0.2778, + "mean_recall_at_4000_tokens": 0.4167, + "mean_recall_at_16000_tokens": 0.4167, + "mean_precision_at_10": 0.0877, + "zero_hit_rate": 0.4286, + "median_tokens_delivered": 3329, + "median_latency_ms": 562.782, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 26, + "mean_recall_at_5": 0.2253, + "full_recall_at_5": 0.1538, + "mean_recall_at_10": 0.3151, + "full_recall_at_10": 0.2308, + "mean_recall_at_20": 0.3151, + "full_recall_at_20": 0.2308, + "mean_recall_at_1000_tokens": 0.0561, + "mean_recall_at_4000_tokens": 0.3151, + "mean_recall_at_16000_tokens": 0.3151, + "mean_precision_at_10": 0.0693, + "zero_hit_rate": 0.5769, + "median_tokens_delivered": 3474, + "median_latency_ms": 607.129, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 47 + }, + "resources": { + "baseline_rss_mb": 16496.5, + "peak_rss_mb": 18366.2, + "delta_rss_mb": 1869.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "jcodemunch", + "summary": { + "all": { + "cases": 47, + "mean_recall_at_5": 0.0736, + "full_recall_at_5": 0.0638, + "mean_recall_at_10": 0.0736, + "full_recall_at_10": 0.0638, + "mean_recall_at_20": 0.0736, + "full_recall_at_20": 0.0638, + "mean_recall_at_1000_tokens": 0.0736, + "mean_recall_at_4000_tokens": 0.0736, + "mean_recall_at_16000_tokens": 0.0736, + "mean_precision_at_10": 0.0461, + "zero_hit_rate": 0.8936, + "median_tokens_delivered": 487, + "median_latency_ms": 741.044, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 1 + }, + "baseline_missed": { + "cases": 40, + "mean_recall_at_5": 0.0615, + "full_recall_at_5": 0.05, + "mean_recall_at_10": 0.0615, + "full_recall_at_10": 0.05, + "mean_recall_at_20": 0.0615, + "full_recall_at_20": 0.05, + "mean_recall_at_1000_tokens": 0.0615, + "mean_recall_at_4000_tokens": 0.0615, + "mean_recall_at_16000_tokens": 0.0615, + "mean_precision_at_10": 0.0417, + "zero_hit_rate": 0.9, + "median_tokens_delivered": 488.5, + "median_latency_ms": 733.925, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 1 + }, + "baseline_solved": { + "cases": 7, + "mean_recall_at_5": 0.1429, + "full_recall_at_5": 0.1429, + "mean_recall_at_10": 0.1429, + "full_recall_at_10": 0.1429, + "mean_recall_at_20": 0.1429, + "full_recall_at_20": 0.1429, + "mean_recall_at_1000_tokens": 0.1429, + "mean_recall_at_4000_tokens": 0.1429, + "mean_recall_at_16000_tokens": 0.1429, + "mean_precision_at_10": 0.0714, + "zero_hit_rate": 0.8571, + "median_tokens_delivered": 482, + "median_latency_ms": 784.281, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.1587, + "full_recall_at_5": 0.1429, + "mean_recall_at_10": 0.1587, + "full_recall_at_10": 0.1429, + "mean_recall_at_20": 0.1587, + "full_recall_at_20": 0.1429, + "mean_recall_at_1000_tokens": 0.1587, + "mean_recall_at_4000_tokens": 0.1587, + "mean_recall_at_16000_tokens": 0.1587, + "mean_precision_at_10": 0.0794, + "zero_hit_rate": 0.8095, + "median_tokens_delivered": 494, + "median_latency_ms": 691.946, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 26, + "mean_recall_at_5": 0.0048, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0048, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0048, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.0048, + "mean_recall_at_4000_tokens": 0.0048, + "mean_recall_at_16000_tokens": 0.0048, + "mean_precision_at_10": 0.0192, + "zero_hit_rate": 0.9615, + "median_tokens_delivered": 484, + "median_latency_ms": 761.3485, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 1 + } + }, + "outcomes": { + "answered": 46, + "indexed-but-returned-nothing": 1 + }, + "resources": { + "baseline_rss_mb": 16931.6, + "peak_rss_mb": 21111.9, + "delta_rss_mb": 4180.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "cocoindex-code", + "summary": { + "all": { + "cases": 47, + "mean_recall_at_5": 0.0603, + "full_recall_at_5": 0.0426, + "mean_recall_at_10": 0.0603, + "full_recall_at_10": 0.0426, + "mean_recall_at_20": 0.0603, + "full_recall_at_20": 0.0426, + "mean_recall_at_1000_tokens": 0.0071, + "mean_recall_at_4000_tokens": 0.0496, + "mean_recall_at_16000_tokens": 0.0603, + "mean_precision_at_10": 0.0197, + "zero_hit_rate": 0.9149, + "median_tokens_delivered": 2607, + "median_latency_ms": 845.779, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 40, + "mean_recall_at_5": 0.0458, + "full_recall_at_5": 0.025, + "mean_recall_at_10": 0.0458, + "full_recall_at_10": 0.025, + "mean_recall_at_20": 0.0458, + "full_recall_at_20": 0.025, + "mean_recall_at_1000_tokens": 0.0083, + "mean_recall_at_4000_tokens": 0.0333, + "mean_recall_at_16000_tokens": 0.0458, + "mean_precision_at_10": 0.02, + "zero_hit_rate": 0.925, + "median_tokens_delivered": 2607, + "median_latency_ms": 862.497, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 7, + "mean_recall_at_5": 0.1429, + "full_recall_at_5": 0.1429, + "mean_recall_at_10": 0.1429, + "full_recall_at_10": 0.1429, + "mean_recall_at_20": 0.1429, + "full_recall_at_20": 0.1429, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1429, + "mean_recall_at_16000_tokens": 0.1429, + "mean_precision_at_10": 0.0179, + "zero_hit_rate": 0.8571, + "median_tokens_delivered": 2607, + "median_latency_ms": 741.196, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.1349, + "full_recall_at_5": 0.0952, + "mean_recall_at_10": 0.1349, + "full_recall_at_10": 0.0952, + "mean_recall_at_20": 0.1349, + "full_recall_at_20": 0.0952, + "mean_recall_at_1000_tokens": 0.0159, + "mean_recall_at_4000_tokens": 0.1111, + "mean_recall_at_16000_tokens": 0.1349, + "mean_precision_at_10": 0.044, + "zero_hit_rate": 0.8095, + "median_tokens_delivered": 2607, + "median_latency_ms": 863.152, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 26, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 2607, + "median_latency_ms": 833.722, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 47 + }, + "resources": { + "baseline_rss_mb": 18997.5, + "peak_rss_mb": 25563.8, + "delta_rss_mb": 6566.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "email-manager-0476e5ea7a56", + "path_leak": true, + "required_files": [ + "src/app/api/auth/[...all]/route.ts" + ], + "found": [], + "missed": [ + "src/app/api/auth/[...all]/route.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82897, + "latency_ms": 25.735, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-0d87af9a34b0", + "path_leak": true, + "required_files": [ + "src/components/GmailFilterBuilder.tsx" + ], + "found": [ + "src/components/GmailFilterBuilder.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 18158, + "latency_ms": 14.203, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-0eddf5a467de", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16175, + "latency_ms": 15.58, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-13cc40c36dae", + "path_leak": false, + "required_files": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "found": [ + "src/lib/embeddings.ts" + ], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 80783, + "latency_ms": 13.541, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-16cd9b84e276", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "next.config.ts", + "wrangler.toml" + ], + "found": [ + "next.config.ts", + "wrangler.toml" + ], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 6886, + "latency_ms": 12.096, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-16da9a4b0f8e", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 43583, + "latency_ms": 12.634, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-1f4a2fcb88d5", + "path_leak": false, + "required_files": [ + "landing-astro/src/pages/index.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/pages/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21416, + "latency_ms": 16.52, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-1fc66e44e95f", + "path_leak": true, + "required_files": [ + "src/app/opengraph-image.tsx" + ], + "found": [], + "missed": [ + "src/app/opengraph-image.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12610, + "latency_ms": 12.294, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-2961ceed2611", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx" + ], + "found": [ + "src/app/app/HomeClient.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 17, + "tokens_delivered": 15243, + "latency_ms": 12.868, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-2aedecaad649", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7796, + "latency_ms": 13.339, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-3338db62bd1b", + "path_leak": false, + "required_files": [ + "landing-astro/src/layouts/Layout.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/layouts/Layout.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18126, + "latency_ms": 16.721, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-33790a7fd4e5", + "path_leak": true, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 14, + "tokens_delivered": 9805, + "latency_ms": 25.847, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-338b73db29bb", + "path_leak": true, + "required_files": [ + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15985, + "latency_ms": 13.128, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-35501ddbd7e8", + "path_leak": true, + "required_files": [ + "scripts/cf-build.mjs" + ], + "found": [ + "scripts/cf-build.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 77583, + "latency_ms": 13.658, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-35efc8549441", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "found": [ + "src/components/posthog-provider.tsx", + "src/pages/HomeClient.tsx" + ], + "missed": [ + "public/_headers", + "src/lib/analytics.ts", + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "tests/open-app.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.125, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.125, + "first_hit_rank": 6, + "tokens_delivered": 51001, + "latency_ms": 13.563, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-3c474e85a025", + "path_leak": true, + "required_files": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "found": [ + "landing-astro/src/pages/faq.astro", + "src/pages/PrivacyPage.tsx" + ], + "missed": [ + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 12547, + "latency_ms": 12.784, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-3cbd63102c96", + "path_leak": true, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8123, + "latency_ms": 17.116, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-45ab8efac77e", + "path_leak": false, + "required_files": [ + "src/components/EmailDetail.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailDetail.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12152, + "latency_ms": 16.086, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-4f284d35f52f", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10212, + "latency_ms": 12.552, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-50270cf8bf36", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailList.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105836, + "latency_ms": 15.515, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-5dd4caedcd8d", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 84224, + "latency_ms": 17.201, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-5e56f1efdc60", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "tsconfig.json" + ], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 78003, + "latency_ms": 16.349, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-61a651298526", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailList.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105836, + "latency_ms": 16.395, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-63eecd0ec045", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "found": [ + "worker.mjs" + ], + "missed": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 110079, + "latency_ms": 15.039, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-64def25acbb3", + "path_leak": true, + "required_files": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "src/lib/auth.ts", + "wrangler.toml" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 79248, + "latency_ms": 17.633, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-6acde8cf1b54", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12295, + "latency_ms": 13.479, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-6cf000313340", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8308, + "latency_ms": 14.755, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-7796afb6ea63", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [ + "wrangler.toml" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 82790, + "latency_ms": 14.411, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-7b0fe5379d1a", + "path_leak": false, + "required_files": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "found": [ + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0.8, + "precision_at_5": 0.8, + "recall_at_10": 0.8, + "precision_at_10": 0.4, + "recall_at_20": 0.8, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.8, + "first_hit_rank": 1, + "tokens_delivered": 40120, + "latency_ms": 13.816, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-88e6e8039b06", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17703, + "latency_ms": 25.355, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-8a3b6110595c", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8549, + "latency_ms": 12.004, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-92f5176080c2", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3864, + "latency_ms": 13.559, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-95a35f10d8b7", + "path_leak": true, + "required_files": [ + "src/worker.ts", + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "found": [ + "wrangler.toml" + ], + "missed": [ + "src/worker.ts", + "tests/open-app.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 72616, + "latency_ms": 14.318, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-ade59255e434", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 129236, + "latency_ms": 14.891, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-af92947d28e7", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5563, + "latency_ms": 14.78, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-b3822a197c25", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12349, + "latency_ms": 15.847, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-b5e31e012879", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6013, + "latency_ms": 15.864, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-ba9e6497f9eb", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7381, + "latency_ms": 14.469, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-c64f6290bdd8", + "path_leak": true, + "required_files": [ + "wrangler.toml" + ], + "found": [ + "wrangler.toml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 76352, + "latency_ms": 13.058, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-c7d7703e1f35", + "path_leak": false, + "required_files": [ + "src/app/page.tsx" + ], + "found": [ + "src/app/page.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 6707, + "latency_ms": 14.479, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-cd131b24cb29", + "path_leak": true, + "required_files": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16304, + "latency_ms": 31.597, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-d1746f53d55e", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "found": [], + "missed": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 75981, + "latency_ms": 13.191, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-e1bbe3e69163", + "path_leak": false, + "required_files": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "found": [ + "src/lib/security-headers.ts" + ], + "missed": [ + "src/worker.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 74523, + "latency_ms": 14.444, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-e52fcd58cd2d", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 42852, + "latency_ms": 11.725, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-edc4fd5d594b", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 9715, + "latency_ms": 13.571, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-f1c2f027bafe", + "path_leak": false, + "required_files": [ + "tests/example.spec.ts" + ], + "found": [], + "missed": [ + "tests/example.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12900, + "latency_ms": 13.463, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-f24464fd6750", + "path_leak": true, + "required_files": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "found": [ + "src/components/TriageQueues.tsx" + ], + "missed": [ + "src/components/EmailDetail.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "recall_at_5": 0.125, + "precision_at_5": 0.2, + "recall_at_10": 0.125, + "precision_at_10": 0.1, + "recall_at_20": 0.125, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.125, + "recall_at_16000_tokens": 0.125, + "first_hit_rank": 3, + "tokens_delivered": 14806, + "latency_ms": 12.691, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-0476e5ea7a56", + "path_leak": true, + "required_files": [ + "src/app/api/auth/[...all]/route.ts" + ], + "found": [], + "missed": [ + "src/app/api/auth/[...all]/route.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86151, + "latency_ms": 23.47, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-0d87af9a34b0", + "path_leak": true, + "required_files": [ + "src/components/GmailFilterBuilder.tsx" + ], + "found": [], + "missed": [ + "src/components/GmailFilterBuilder.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 114507, + "latency_ms": 26.823, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-0eddf5a467de", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 92915, + "latency_ms": 27.169, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-13cc40c36dae", + "path_leak": false, + "required_files": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "found": [ + "next.config.ts" + ], + "missed": [ + "src/lib/embeddings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 81053, + "latency_ms": 22.179, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-16cd9b84e276", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "next.config.ts", + "wrangler.toml" + ], + "found": [ + "next.config.ts", + "wrangler.toml" + ], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 78112, + "latency_ms": 22.741, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-16da9a4b0f8e", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 43583, + "latency_ms": 21.321, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-1f4a2fcb88d5", + "path_leak": false, + "required_files": [ + "landing-astro/src/pages/index.astro" + ], + "found": [ + "landing-astro/src/pages/index.astro" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 136353, + "latency_ms": 43.926, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-1fc66e44e95f", + "path_leak": true, + "required_files": [ + "src/app/opengraph-image.tsx" + ], + "found": [], + "missed": [ + "src/app/opengraph-image.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 88809, + "latency_ms": 24.815, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-2961ceed2611", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx" + ], + "found": [], + "missed": [ + "src/app/app/HomeClient.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 114539, + "latency_ms": 23.156, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-2aedecaad649", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [ + "wrangler.toml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 83883, + "latency_ms": 22.084, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-3338db62bd1b", + "path_leak": false, + "required_files": [ + "landing-astro/src/layouts/Layout.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/layouts/Layout.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 136353, + "latency_ms": 28.468, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-33790a7fd4e5", + "path_leak": true, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json", + "tsconfig.json" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 46302, + "latency_ms": 21.9, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-338b73db29bb", + "path_leak": true, + "required_files": [ + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 139417, + "latency_ms": 31.947, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-35501ddbd7e8", + "path_leak": true, + "required_files": [ + "scripts/cf-build.mjs" + ], + "found": [], + "missed": [ + "scripts/cf-build.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 77940, + "latency_ms": 22.762, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-35efc8549441", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "found": [ + "src/lib/foundry-monitoring.ts", + "src/pages/HomeClient.tsx" + ], + "missed": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "tests/open-app.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 90883, + "latency_ms": 24.425, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-3c474e85a025", + "path_leak": true, + "required_files": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "found": [], + "missed": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 135232, + "latency_ms": 27.573, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-3cbd63102c96", + "path_leak": true, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 83842, + "latency_ms": 23.056, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-45ab8efac77e", + "path_leak": false, + "required_files": [ + "src/components/EmailDetail.tsx" + ], + "found": [ + "src/components/EmailDetail.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 114463, + "latency_ms": 20.942, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-4f284d35f52f", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 46156, + "latency_ms": 23.834, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-50270cf8bf36", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [ + "src/components/EmailList.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 114463, + "latency_ms": 23.353, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-5dd4caedcd8d", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 84238, + "latency_ms": 42.054, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-5e56f1efdc60", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json", + "tsconfig.json" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 80214, + "latency_ms": 24.215, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-61a651298526", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [ + "src/components/EmailList.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 114463, + "latency_ms": 19.56, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-63eecd0ec045", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "found": [], + "missed": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 113173, + "latency_ms": 33.005, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-64def25acbb3", + "path_leak": true, + "required_files": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "src/lib/auth.ts", + "wrangler.toml" + ], + "found": [ + "src/lib/auth.ts", + "wrangler.toml" + ], + "missed": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 83842, + "latency_ms": 22.268, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-6acde8cf1b54", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [ + "next.config.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 78526, + "latency_ms": 23.275, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-6cf000313340", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 83868, + "latency_ms": 22.429, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-7796afb6ea63", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [ + "wrangler.toml" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 81053, + "latency_ms": 35.748, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-7b0fe5379d1a", + "path_leak": false, + "required_files": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "found": [ + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.8, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 6, + "tokens_delivered": 40120, + "latency_ms": 22.375, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-88e6e8039b06", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 139417, + "latency_ms": 26.142, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-8a3b6110595c", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 83901, + "latency_ms": 21.564, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-92f5176080c2", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 83844, + "latency_ms": 22.98, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-95a35f10d8b7", + "path_leak": true, + "required_files": [ + "src/worker.ts", + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "found": [ + "src/worker.ts", + "wrangler.toml" + ], + "missed": [ + "tests/open-app.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 90651, + "latency_ms": 23.218, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-ade59255e434", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 141342, + "latency_ms": 28.72, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-af92947d28e7", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 83901, + "latency_ms": 24.229, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-b3822a197c25", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [ + "next.config.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88809, + "latency_ms": 24.83, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-b5e31e012879", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 81051, + "latency_ms": 21.293, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-ba9e6497f9eb", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [ + "next.config.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 77903, + "latency_ms": 36.46, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-c64f6290bdd8", + "path_leak": true, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 78303, + "latency_ms": 39.548, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-c7d7703e1f35", + "path_leak": false, + "required_files": [ + "src/app/page.tsx" + ], + "found": [ + "src/app/page.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 83818, + "latency_ms": 22.734, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-cd131b24cb29", + "path_leak": true, + "required_files": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140021, + "latency_ms": 24.247, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-d1746f53d55e", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "found": [], + "missed": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 92912, + "latency_ms": 26.898, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-e1bbe3e69163", + "path_leak": false, + "required_files": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "found": [], + "missed": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 88536, + "latency_ms": 23.857, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-e52fcd58cd2d", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 42852, + "latency_ms": 24.07, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-edc4fd5d594b", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 78587, + "latency_ms": 27.235, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-f1c2f027bafe", + "path_leak": false, + "required_files": [ + "tests/example.spec.ts" + ], + "found": [], + "missed": [ + "tests/example.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 113080, + "latency_ms": 28.02, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-f24464fd6750", + "path_leak": true, + "required_files": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "found": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/worker.ts" + ], + "missed": [ + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css" + ], + "recall_at_5": 0.125, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.2, + "recall_at_20": 0.375, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 91399, + "latency_ms": 27.682, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-0476e5ea7a56", + "path_leak": true, + "required_files": [ + "src/app/api/auth/[...all]/route.ts" + ], + "found": [], + "missed": [ + "src/app/api/auth/[...all]/route.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3636, + "latency_ms": 518.663, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-0d87af9a34b0", + "path_leak": true, + "required_files": [ + "src/components/GmailFilterBuilder.tsx" + ], + "found": [], + "missed": [ + "src/components/GmailFilterBuilder.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3547, + "latency_ms": 623.378, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-0eddf5a467de", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3740, + "latency_ms": 790.629, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-13cc40c36dae", + "path_leak": false, + "required_files": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "found": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2838, + "latency_ms": 474.948, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-16cd9b84e276", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "next.config.ts", + "wrangler.toml" + ], + "found": [ + "next.config.ts" + ], + "missed": [ + ".husky/pre-push", + "wrangler.toml" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 2800, + "latency_ms": 536.031, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-16da9a4b0f8e", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3008, + "latency_ms": 459.748, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-1f4a2fcb88d5", + "path_leak": false, + "required_files": [ + "landing-astro/src/pages/index.astro" + ], + "found": [ + "landing-astro/src/pages/index.astro" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3512, + "latency_ms": 724.872, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-1fc66e44e95f", + "path_leak": true, + "required_files": [ + "src/app/opengraph-image.tsx" + ], + "found": [], + "missed": [ + "src/app/opengraph-image.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3431, + "latency_ms": 562.782, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-2961ceed2611", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx" + ], + "found": [], + "missed": [ + "src/app/app/HomeClient.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3640, + "latency_ms": 595.854, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-2aedecaad649", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3519, + "latency_ms": 523.262, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-3338db62bd1b", + "path_leak": false, + "required_files": [ + "landing-astro/src/layouts/Layout.astro" + ], + "found": [ + "landing-astro/src/layouts/Layout.astro" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 5441, + "latency_ms": 808.15, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-33790a7fd4e5", + "path_leak": true, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2823, + "latency_ms": 508.625, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-338b73db29bb", + "path_leak": true, + "required_files": [ + "scripts/verify-public-indexing.mjs" + ], + "found": [ + "scripts/verify-public-indexing.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3289, + "latency_ms": 789.196, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-35501ddbd7e8", + "path_leak": true, + "required_files": [ + "scripts/cf-build.mjs" + ], + "found": [ + "scripts/cf-build.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3329, + "latency_ms": 551.908, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-35efc8549441", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "found": [ + "src/lib/foundry-monitoring.ts" + ], + "missed": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "recall_at_5": 0.125, + "precision_at_5": 0.2, + "recall_at_10": 0.125, + "precision_at_10": 0.1667, + "recall_at_20": 0.125, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.125, + "recall_at_4000_tokens": 0.125, + "recall_at_16000_tokens": 0.125, + "first_hit_rank": 1, + "tokens_delivered": 3377, + "latency_ms": 792.251, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-3c474e85a025", + "path_leak": true, + "required_files": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "found": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro" + ], + "missed": [ + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3970, + "latency_ms": 784.659, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-3cbd63102c96", + "path_leak": true, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3795, + "latency_ms": 563.12, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-45ab8efac77e", + "path_leak": false, + "required_files": [ + "src/components/EmailDetail.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailDetail.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3816, + "latency_ms": 742.32, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-4f284d35f52f", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2767, + "latency_ms": 548.84, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-50270cf8bf36", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailList.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3199, + "latency_ms": 664.176, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-5dd4caedcd8d", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3679, + "latency_ms": 554.377, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-5e56f1efdc60", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2640, + "latency_ms": 503.477, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-61a651298526", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailList.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3199, + "latency_ms": 660.216, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-63eecd0ec045", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "found": [ + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "missed": [ + "src/app/app/HomeClient.tsx" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 3760, + "latency_ms": 650.36, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-64def25acbb3", + "path_leak": true, + "required_files": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "src/lib/auth.ts", + "wrangler.toml" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "wrangler.toml" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 3088, + "latency_ms": 524.422, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-6acde8cf1b54", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [ + "next.config.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3662, + "latency_ms": 515.25, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-6cf000313340", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3030, + "latency_ms": 640.341, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-7796afb6ea63", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3220, + "latency_ms": 517.562, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-7b0fe5379d1a", + "path_leak": false, + "required_files": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "found": [ + "src/app/api/emails/route.ts", + "src/lib/auth.ts" + ], + "missed": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/page.tsx" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.4, + "recall_at_20": 0.4, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 2, + "tokens_delivered": 3137, + "latency_ms": 516.637, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-88e6e8039b06", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3710, + "latency_ms": 790.244, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-8a3b6110595c", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2817, + "latency_ms": 516.997, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-92f5176080c2", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2765, + "latency_ms": 587.92, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-95a35f10d8b7", + "path_leak": true, + "required_files": [ + "src/worker.ts", + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "found": [ + "src/worker.ts" + ], + "missed": [ + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.125, + "recall_at_20": 0.3333, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 3526, + "latency_ms": 746.133, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-ade59255e434", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3695, + "latency_ms": 800.863, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-af92947d28e7", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3152, + "latency_ms": 759.974, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-b3822a197c25", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [ + "next.config.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3448, + "latency_ms": 1121.891, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-b5e31e012879", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3493, + "latency_ms": 539.083, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-ba9e6497f9eb", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [ + "next.config.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3076, + "latency_ms": 474.029, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-c64f6290bdd8", + "path_leak": true, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3301, + "latency_ms": 543.618, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-c7d7703e1f35", + "path_leak": false, + "required_files": [ + "src/app/page.tsx" + ], + "found": [], + "missed": [ + "src/app/page.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2720, + "latency_ms": 538.543, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-cd131b24cb29", + "path_leak": true, + "required_files": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "found": [ + "landing-astro/src/layouts/Layout.astro" + ], + "missed": [ + "index.html", + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 6, + "tokens_delivered": 5499, + "latency_ms": 759.652, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-d1746f53d55e", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "found": [ + "src/lib/security-headers.ts" + ], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 3659, + "latency_ms": 847.971, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-e1bbe3e69163", + "path_leak": false, + "required_files": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "found": [ + "src/worker.ts" + ], + "missed": [ + "src/lib/security-headers.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3458, + "latency_ms": 821.625, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-e52fcd58cd2d", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3326, + "latency_ms": 544.816, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-edc4fd5d594b", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3490, + "latency_ms": 542.902, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-f1c2f027bafe", + "path_leak": false, + "required_files": [ + "tests/example.spec.ts" + ], + "found": [], + "missed": [ + "tests/example.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3537, + "latency_ms": 618.404, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "email-manager-f24464fd6750", + "path_leak": true, + "required_files": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "found": [], + "missed": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3685, + "latency_ms": 730.261, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-0476e5ea7a56", + "path_leak": true, + "required_files": [ + "src/app/api/auth/[...all]/route.ts" + ], + "found": [], + "missed": [ + "src/app/api/auth/[...all]/route.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 512, + "latency_ms": 628.43, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-0d87af9a34b0", + "path_leak": true, + "required_files": [ + "src/components/GmailFilterBuilder.tsx" + ], + "found": [], + "missed": [ + "src/components/GmailFilterBuilder.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 466, + "latency_ms": 691.284, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-0eddf5a467de", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 494, + "latency_ms": 691.946, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-13cc40c36dae", + "path_leak": false, + "required_files": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "found": [], + "missed": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 490, + "latency_ms": 596.332, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-16cd9b84e276", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "next.config.ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + ".husky/pre-push", + "next.config.ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 500, + "latency_ms": 636.596, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-16da9a4b0f8e", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 423, + "latency_ms": 458.479, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-1f4a2fcb88d5", + "path_leak": false, + "required_files": [ + "landing-astro/src/pages/index.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/pages/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 626, + "latency_ms": 751.618, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-1fc66e44e95f", + "path_leak": true, + "required_files": [ + "src/app/opengraph-image.tsx" + ], + "found": [ + "src/app/opengraph-image.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 563, + "latency_ms": 603.774, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-2961ceed2611", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx" + ], + "found": [], + "missed": [ + "src/app/app/HomeClient.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 111, + "latency_ms": 699.244, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-2aedecaad649", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 602, + "latency_ms": 612.427, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-3338db62bd1b", + "path_leak": false, + "required_files": [ + "landing-astro/src/layouts/Layout.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/layouts/Layout.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 453, + "latency_ms": 940.321, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-33790a7fd4e5", + "path_leak": true, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 506, + "latency_ms": 526.219, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-338b73db29bb", + "path_leak": true, + "required_files": [ + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 439, + "latency_ms": 986.149, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-35501ddbd7e8", + "path_leak": true, + "required_files": [ + "scripts/cf-build.mjs" + ], + "found": [], + "missed": [ + "scripts/cf-build.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 522, + "latency_ms": 680.346, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-35efc8549441", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "found": [ + "src/lib/foundry-monitoring.ts" + ], + "missed": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "recall_at_5": 0.125, + "precision_at_5": 0.5, + "recall_at_10": 0.125, + "precision_at_10": 0.5, + "recall_at_20": 0.125, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.125, + "recall_at_4000_tokens": 0.125, + "recall_at_16000_tokens": 0.125, + "first_hit_rank": 1, + "tokens_delivered": 440, + "latency_ms": 621.584, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-3c474e85a025", + "path_leak": true, + "required_files": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "found": [], + "missed": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 411, + "latency_ms": 772.649, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-3cbd63102c96", + "path_leak": true, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 368, + "latency_ms": 682.177, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-45ab8efac77e", + "path_leak": false, + "required_files": [ + "src/components/EmailDetail.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailDetail.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 548, + "latency_ms": 841.349, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-4f284d35f52f", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 562, + "latency_ms": 523.359, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-50270cf8bf36", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailList.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 569, + "latency_ms": 734.898, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-5dd4caedcd8d", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 602, + "latency_ms": 968.464, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-5e56f1efdc60", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 429, + "latency_ms": 646.713, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-61a651298526", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailList.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 569, + "latency_ms": 761.423, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-63eecd0ec045", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "found": [], + "missed": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 481, + "latency_ms": 776.915, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-64def25acbb3", + "path_leak": true, + "required_files": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "src/lib/auth.ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "src/lib/auth.ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 602, + "latency_ms": 720.253, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-6acde8cf1b54", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 546, + "latency_ms": 746.712, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-6cf000313340", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 498, + "latency_ms": 676.888, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-7796afb6ea63", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 425, + "latency_ms": 741.044, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-7b0fe5379d1a", + "path_leak": false, + "required_files": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 410, + "latency_ms": 543.537, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-88e6e8039b06", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 259, + "latency_ms": 944.506, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-8a3b6110595c", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 499, + "latency_ms": 773.011, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-92f5176080c2", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 453, + "latency_ms": 809.629, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-95a35f10d8b7", + "path_leak": true, + "required_files": [ + "src/worker.ts", + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "found": [ + "src/worker.ts" + ], + "missed": [ + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 462, + "latency_ms": 697.353, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-ade59255e434", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 608, + "latency_ms": 948.478, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-af92947d28e7", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 487, + "latency_ms": 753.31, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-b3822a197c25", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 550, + "latency_ms": 732.292, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-b5e31e012879", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 509, + "latency_ms": 769.387, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-ba9e6497f9eb", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 490, + "latency_ms": 732.952, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-c64f6290bdd8", + "path_leak": true, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 482, + "latency_ms": 784.281, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-c7d7703e1f35", + "path_leak": false, + "required_files": [ + "src/app/page.tsx" + ], + "found": [], + "missed": [ + "src/app/page.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 873, + "latency_ms": 863.662, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-cd131b24cb29", + "path_leak": true, + "required_files": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 429, + "latency_ms": 1036.451, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-d1746f53d55e", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "found": [], + "missed": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 471, + "latency_ms": 873.093, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-e1bbe3e69163", + "path_leak": false, + "required_files": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "found": [], + "missed": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 341, + "latency_ms": 939.039, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-e52fcd58cd2d", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 461, + "latency_ms": 684.124, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-edc4fd5d594b", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 452, + "latency_ms": 872.499, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-f1c2f027bafe", + "path_leak": false, + "required_files": [ + "tests/example.spec.ts" + ], + "found": [], + "missed": [ + "tests/example.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 396, + "latency_ms": 996.51, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "email-manager-f24464fd6750", + "path_leak": true, + "required_files": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "found": [], + "missed": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 441, + "latency_ms": 932.353, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-0476e5ea7a56", + "path_leak": true, + "required_files": [ + "src/app/api/auth/[...all]/route.ts" + ], + "found": [], + "missed": [ + "src/app/api/auth/[...all]/route.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5330, + "latency_ms": 67722.716, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-0d87af9a34b0", + "path_leak": true, + "required_files": [ + "src/components/GmailFilterBuilder.tsx" + ], + "found": [], + "missed": [ + "src/components/GmailFilterBuilder.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4822, + "latency_ms": 9617.473, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-0eddf5a467de", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4609, + "latency_ms": 15140.964, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-13cc40c36dae", + "path_leak": false, + "required_files": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "found": [], + "missed": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5122, + "latency_ms": 122810.727, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-16cd9b84e276", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "next.config.ts", + "wrangler.toml" + ], + "found": [ + "wrangler.toml" + ], + "missed": [ + ".husky/pre-push", + "next.config.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 4846, + "latency_ms": 2380.336, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-16da9a4b0f8e", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 5314, + "latency_ms": 15876.922, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-1f4a2fcb88d5", + "path_leak": false, + "required_files": [ + "landing-astro/src/pages/index.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/pages/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5051, + "latency_ms": 119325.648, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-1fc66e44e95f", + "path_leak": true, + "required_files": [ + "src/app/opengraph-image.tsx" + ], + "found": [ + "src/app/opengraph-image.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 4627, + "latency_ms": 18903.306, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-2961ceed2611", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx" + ], + "found": [], + "missed": [ + "src/app/app/HomeClient.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4677, + "latency_ms": 6135.991, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-2aedecaad649", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4862, + "latency_ms": 5349.082, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-3338db62bd1b", + "path_leak": false, + "required_files": [ + "landing-astro/src/layouts/Layout.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/layouts/Layout.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5549, + "latency_ms": 59423.29, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-33790a7fd4e5", + "path_leak": true, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "tsconfig.json" + ], + "missed": [ + "package.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 5107, + "latency_ms": 18732.481, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-338b73db29bb", + "path_leak": true, + "required_files": [ + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5191, + "latency_ms": 182015.636, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-35501ddbd7e8", + "path_leak": true, + "required_files": [ + "scripts/cf-build.mjs" + ], + "found": [], + "missed": [ + "scripts/cf-build.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 738.722, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-35efc8549441", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "found": [], + "missed": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 860.054, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-3c474e85a025", + "path_leak": true, + "required_files": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "found": [], + "missed": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 954.469, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-3cbd63102c96", + "path_leak": true, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 845.779, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-45ab8efac77e", + "path_leak": false, + "required_files": [ + "src/components/EmailDetail.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailDetail.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 965.584, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-4f284d35f52f", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1157.807, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-50270cf8bf36", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailList.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 345, + "latency_ms": 784.932, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-5dd4caedcd8d", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 782.466, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-5e56f1efdc60", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 758.065, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-61a651298526", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailList.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 345, + "latency_ms": 659.683, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-63eecd0ec045", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "found": [], + "missed": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 782.769, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-64def25acbb3", + "path_leak": true, + "required_files": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "src/lib/auth.ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "src/lib/auth.ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 783.035, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-6acde8cf1b54", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 740.922, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-6cf000313340", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 743.139, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-7796afb6ea63", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 685.591, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-7b0fe5379d1a", + "path_leak": false, + "required_files": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 789.952, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-88e6e8039b06", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 861.842, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-8a3b6110595c", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 912.557, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-92f5176080c2", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 739.583, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-95a35f10d8b7", + "path_leak": true, + "required_files": [ + "src/worker.ts", + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + "src/worker.ts", + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 800.316, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-ade59255e434", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 837.646, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-af92947d28e7", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1009.158, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-b3822a197c25", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 789.809, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-b5e31e012879", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 642.532, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-ba9e6497f9eb", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 961.517, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-c64f6290bdd8", + "path_leak": true, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 741.196, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-c7d7703e1f35", + "path_leak": false, + "required_files": [ + "src/app/page.tsx" + ], + "found": [], + "missed": [ + "src/app/page.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 829.798, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-cd131b24cb29", + "path_leak": true, + "required_files": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 694.762, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-d1746f53d55e", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "found": [], + "missed": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 751.66, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-e1bbe3e69163", + "path_leak": false, + "required_files": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "found": [], + "missed": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 787.496, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-e52fcd58cd2d", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [], + "missed": [ + "src/lib/gmail.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 797.084, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-edc4fd5d594b", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 938.887, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-f1c2f027bafe", + "path_leak": false, + "required_files": [ + "tests/example.spec.ts" + ], + "found": [], + "missed": [ + "tests/example.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 632.195, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "email-manager-f24464fd6750", + "path_leak": true, + "required_files": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "found": [], + "missed": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 863.152, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/P-protein-index.json b/benchmarks/context-retrieval/results/P-protein-index.json new file mode 100644 index 00000000..2050f983 --- /dev/null +++ b/benchmarks/context-retrieval/results/P-protein-index.json @@ -0,0 +1,8113 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "protein-index", + "head": "e96ee7fd3164bee1493925dcba8a160c5046e1d8" + }, + "tier": "small", + "tier_code_files": 104, + "corpus_counts": { + "cases": 52, + "path_leak": 11, + "baseline_blind": 41, + "multi_file": 35 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.5234, + "control": 0.0144, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "reject volume-basis label candidates", + "revision": "c07e964dd0d26f06c01dbaf0b5f47c94aa87ff05", + "returned": [ + "shared/classification.ts", + "test/api.worker.test.ts", + "migrations/0005_evidence_decisions.sql", + "openspec/changes/build-catalog-core/.openspec.yaml", + "review-decisions/review-615f9e122d922268afd3/manifest.json" + ] + }, + { + "provider_id": "random-files", + "query": "preserve label sugar and calorie semantics", + "revision": "e79743cf9d782a065e0e4a54720d8c5c14982026", + "returned": [ + ".github/workflows/extract-robotoff-ingredients.yml", + "scripts/adapters/robotoff.ts", + "review-decisions/review-4c07ab1d3adc20a99ccb/manifest.json", + "scripts/adapters/robotoff-ingredients.ts", + "openspec/config.yaml" + ] + }, + { + "provider_id": "random-files", + "query": "publish source catalog to D1", + "revision": "c339b464b206cb32f5966ec8240c0e79375f9674", + "returned": [ + "scripts/fixtures.ts", + "test/domain.test.ts", + "vite.config.ts", + "package.json", + "worker/catalog.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "reject volume-basis label candidates", + "revision": "c07e964dd0d26f06c01dbaf0b5f47c94aa87ff05", + "returned": [ + "test/ingestion.test.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "preserve unknown nutrition basis", + "revision": "392d519207f9163a413d8e09d37305866f595067", + "returned": [ + "test/ingestion.test.ts", + "scripts/reconcile.ts", + "src/App.tsx", + ".github/workflows/source-sync.yml", + "scripts/adapters/open-food-facts-api.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "select current review evidence", + "revision": "cadcf1bd67cfddbe126fb06f5204e180e5f25be4", + "returned": [ + "test/ingestion.test.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "scripts/reconcile.ts", + "src/App.tsx" + ] + }, + { + "provider_id": "semble", + "query": "reject volume-basis label candidates", + "revision": "c07e964dd0d26f06c01dbaf0b5f47c94aa87ff05", + "returned": [ + "scripts/adapters/robotoff-ingredients.ts", + "scripts/adapters/robotoff.ts", + "shared/evidence-decisions.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "shared/ingredient-evidence.ts" + ] + }, + { + "provider_id": "semble", + "query": "parse current MuscleBlaze catalog pages", + "revision": "27c937b786737b5f3a6f9be31b89622f8ec49d5d", + "returned": [ + "worker/catalog.ts", + "shared/api.ts", + "src/api.ts", + "worker/completion.ts", + "worker/agent-index.ts" + ] + }, + { + "provider_id": "semble", + "query": "publish source catalog to D1", + "revision": "c339b464b206cb32f5966ec8240c0e79375f9674", + "returned": [ + "worker/catalog.ts", + "scripts/sync.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/fixtures.ts", + "worker/coverage.ts" + ] + }, + { + "provider_id": "jcodemunch", + "query": "withhold implausible protein metrics", + "revision": "5c62a39276da010bfd4aab9958bd9a3f2548c802", + "returned": [ + "shared/types.ts", + "test/api.worker.test.ts", + "src/styles.css" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 52, + "mean_recall_at_5": 0.0503, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1006, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1638, + "full_recall_at_20": 0.0385, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0096, + "mean_recall_at_16000_tokens": 0.0407, + "mean_precision_at_10": 0.0288, + "zero_hit_rate": 0.6538, + "median_tokens_delivered": 40103.5, + "median_latency_ms": 14.8455, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 52, + "mean_recall_at_5": 0.0503, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1006, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1638, + "full_recall_at_20": 0.0385, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0096, + "mean_recall_at_16000_tokens": 0.0407, + "mean_precision_at_10": 0.0288, + "zero_hit_rate": 0.6538, + "median_tokens_delivered": 40103.5, + "median_latency_ms": 14.8455, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 11, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0152, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0606, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0152, + "mean_precision_at_10": 0.0091, + "zero_hit_rate": 0.7273, + "median_tokens_delivered": 43785, + "median_latency_ms": 14.1, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 41, + "mean_recall_at_5": 0.0638, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1236, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1915, + "full_recall_at_20": 0.0488, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0122, + "mean_recall_at_16000_tokens": 0.0476, + "mean_precision_at_10": 0.0341, + "zero_hit_rate": 0.6341, + "median_tokens_delivered": 40044, + "median_latency_ms": 14.952, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 52 + }, + "resources": { + "baseline_rss_mb": 16846.1, + "peak_rss_mb": 17147.1, + "delta_rss_mb": 301.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 52, + "mean_recall_at_5": 0.3603, + "full_recall_at_5": 0.1538, + "mean_recall_at_10": 0.4788, + "full_recall_at_10": 0.2308, + "mean_recall_at_20": 0.5651, + "full_recall_at_20": 0.3462, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0144, + "mean_recall_at_16000_tokens": 0.2883, + "mean_precision_at_10": 0.1346, + "zero_hit_rate": 0.2885, + "median_tokens_delivered": 145928, + "median_latency_ms": 27.327, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 52, + "mean_recall_at_5": 0.3603, + "full_recall_at_5": 0.1538, + "mean_recall_at_10": 0.4788, + "full_recall_at_10": 0.2308, + "mean_recall_at_20": 0.5651, + "full_recall_at_20": 0.3462, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0144, + "mean_recall_at_16000_tokens": 0.2883, + "mean_precision_at_10": 0.1346, + "zero_hit_rate": 0.2885, + "median_tokens_delivered": 145928, + "median_latency_ms": 27.327, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 11, + "mean_recall_at_5": 0.2682, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.3515, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.4727, + "full_recall_at_20": 0.1818, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.2076, + "mean_precision_at_10": 0.1364, + "zero_hit_rate": 0.2727, + "median_tokens_delivered": 189454, + "median_latency_ms": 28.919, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 41, + "mean_recall_at_5": 0.385, + "full_recall_at_5": 0.1951, + "mean_recall_at_10": 0.513, + "full_recall_at_10": 0.2927, + "mean_recall_at_20": 0.5898, + "full_recall_at_20": 0.3902, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0183, + "mean_recall_at_16000_tokens": 0.31, + "mean_precision_at_10": 0.1341, + "zero_hit_rate": 0.2927, + "median_tokens_delivered": 140313, + "median_latency_ms": 27.115, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 52 + }, + "resources": { + "baseline_rss_mb": 16255.9, + "peak_rss_mb": 16675.3, + "delta_rss_mb": 419.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 52, + "mean_recall_at_5": 0.2646, + "full_recall_at_5": 0.0577, + "mean_recall_at_10": 0.4188, + "full_recall_at_10": 0.1154, + "mean_recall_at_20": 0.533, + "full_recall_at_20": 0.2115, + "mean_recall_at_1000_tokens": 0.2361, + "mean_recall_at_4000_tokens": 0.5234, + "mean_recall_at_16000_tokens": 0.533, + "mean_precision_at_10": 0.1139, + "zero_hit_rate": 0.2308, + "median_tokens_delivered": 3652.5, + "median_latency_ms": 1710.6305, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 52, + "mean_recall_at_5": 0.2646, + "full_recall_at_5": 0.0577, + "mean_recall_at_10": 0.4188, + "full_recall_at_10": 0.1154, + "mean_recall_at_20": 0.533, + "full_recall_at_20": 0.2115, + "mean_recall_at_1000_tokens": 0.2361, + "mean_recall_at_4000_tokens": 0.5234, + "mean_recall_at_16000_tokens": 0.533, + "mean_precision_at_10": 0.1139, + "zero_hit_rate": 0.2308, + "median_tokens_delivered": 3652.5, + "median_latency_ms": 1710.6305, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 11, + "mean_recall_at_5": 0.4924, + "full_recall_at_5": 0.1818, + "mean_recall_at_10": 0.5985, + "full_recall_at_10": 0.1818, + "mean_recall_at_20": 0.6288, + "full_recall_at_20": 0.1818, + "mean_recall_at_1000_tokens": 0.4621, + "mean_recall_at_4000_tokens": 0.6288, + "mean_recall_at_16000_tokens": 0.6288, + "mean_precision_at_10": 0.1657, + "zero_hit_rate": 0.0909, + "median_tokens_delivered": 3771, + "median_latency_ms": 1806.216, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 41, + "mean_recall_at_5": 0.2035, + "full_recall_at_5": 0.0244, + "mean_recall_at_10": 0.3705, + "full_recall_at_10": 0.0976, + "mean_recall_at_20": 0.5073, + "full_recall_at_20": 0.2195, + "mean_recall_at_1000_tokens": 0.1754, + "mean_recall_at_4000_tokens": 0.4951, + "mean_recall_at_16000_tokens": 0.5073, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.2683, + "median_tokens_delivered": 3652, + "median_latency_ms": 1670.04, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 52 + }, + "resources": { + "baseline_rss_mb": 17041.3, + "peak_rss_mb": 20143.1, + "delta_rss_mb": 3101.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "jcodemunch", + "summary": { + "all": { + "cases": 52, + "mean_recall_at_5": 0.2263, + "full_recall_at_5": 0.1154, + "mean_recall_at_10": 0.2263, + "full_recall_at_10": 0.1154, + "mean_recall_at_20": 0.2263, + "full_recall_at_20": 0.1154, + "mean_recall_at_1000_tokens": 0.2263, + "mean_recall_at_4000_tokens": 0.2263, + "mean_recall_at_16000_tokens": 0.2263, + "mean_precision_at_10": 0.2276, + "zero_hit_rate": 0.5769, + "median_tokens_delivered": 582.5, + "median_latency_ms": 676.5815, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 1 + }, + "baseline_missed": { + "cases": 52, + "mean_recall_at_5": 0.2263, + "full_recall_at_5": 0.1154, + "mean_recall_at_10": 0.2263, + "full_recall_at_10": 0.1154, + "mean_recall_at_20": 0.2263, + "full_recall_at_20": 0.1154, + "mean_recall_at_1000_tokens": 0.2263, + "mean_recall_at_4000_tokens": 0.2263, + "mean_recall_at_16000_tokens": 0.2263, + "mean_precision_at_10": 0.2276, + "zero_hit_rate": 0.5769, + "median_tokens_delivered": 582.5, + "median_latency_ms": 676.5815, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 1 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 11, + "mean_recall_at_5": 0.3364, + "full_recall_at_5": 0.1818, + "mean_recall_at_10": 0.3364, + "full_recall_at_10": 0.1818, + "mean_recall_at_20": 0.3364, + "full_recall_at_20": 0.1818, + "mean_recall_at_1000_tokens": 0.3364, + "mean_recall_at_4000_tokens": 0.3364, + "mean_recall_at_16000_tokens": 0.3364, + "mean_precision_at_10": 0.3333, + "zero_hit_rate": 0.3636, + "median_tokens_delivered": 591, + "median_latency_ms": 680.118, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 41, + "mean_recall_at_5": 0.1967, + "full_recall_at_5": 0.0976, + "mean_recall_at_10": 0.1967, + "full_recall_at_10": 0.0976, + "mean_recall_at_20": 0.1967, + "full_recall_at_20": 0.0976, + "mean_recall_at_1000_tokens": 0.1967, + "mean_recall_at_4000_tokens": 0.1967, + "mean_recall_at_16000_tokens": 0.1967, + "mean_precision_at_10": 0.1992, + "zero_hit_rate": 0.6341, + "median_tokens_delivered": 569, + "median_latency_ms": 673.045, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 1 + } + }, + "outcomes": { + "answered": 51, + "indexed-but-returned-nothing": 1 + }, + "resources": { + "baseline_rss_mb": 18462.8, + "peak_rss_mb": 21142.9, + "delta_rss_mb": 2680.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "cocoindex-code", + "summary": { + "all": { + "cases": 52, + "mean_recall_at_5": 0.0103, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0341, + "full_recall_at_10": 0.0192, + "mean_recall_at_20": 0.0341, + "full_recall_at_20": 0.0192, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0341, + "mean_recall_at_16000_tokens": 0.0341, + "mean_precision_at_10": 0.0163, + "zero_hit_rate": 0.9423, + "median_tokens_delivered": 2607, + "median_latency_ms": 731.679, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 52, + "mean_recall_at_5": 0.0103, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0341, + "full_recall_at_10": 0.0192, + "mean_recall_at_20": 0.0341, + "full_recall_at_20": 0.0192, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0341, + "mean_recall_at_16000_tokens": 0.0341, + "mean_precision_at_10": 0.0163, + "zero_hit_rate": 0.9423, + "median_tokens_delivered": 2607, + "median_latency_ms": 731.679, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 11, + "mean_recall_at_5": 0.0303, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0909, + "full_recall_at_10": 0.0909, + "mean_recall_at_20": 0.0909, + "full_recall_at_20": 0.0909, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0909, + "mean_recall_at_16000_tokens": 0.0909, + "mean_precision_at_10": 0.0273, + "zero_hit_rate": 0.9091, + "median_tokens_delivered": 2607, + "median_latency_ms": 775.92, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 41, + "mean_recall_at_5": 0.0049, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0189, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0189, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0189, + "mean_recall_at_16000_tokens": 0.0189, + "mean_precision_at_10": 0.0134, + "zero_hit_rate": 0.9512, + "median_tokens_delivered": 2607, + "median_latency_ms": 731.644, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 52 + }, + "resources": { + "baseline_rss_mb": 20173.5, + "peak_rss_mb": 25549.8, + "delta_rss_mb": 5376.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "protein-index-02bae53a54b2", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "vitest.worker.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 209011, + "latency_ms": 27.923, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-12524e50831f", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28922, + "latency_ms": 15.401, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-14a6c420d60c", + "path_leak": true, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13328, + "latency_ms": 13.359, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-183e0a12fb52", + "path_leak": true, + "required_files": [ + "scripts/machine-label.ts" + ], + "found": [], + "missed": [ + "scripts/machine-label.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 192434, + "latency_ms": 14.849, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-196a41597b98", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/domain.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 163973, + "latency_ms": 12.163, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-1a649b8e8091", + "path_leak": false, + "required_files": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40044, + "latency_ms": 14.933, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-1af57b34dd19", + "path_leak": true, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63853, + "latency_ms": 12.639, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-1e4d6787f0ec", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 167953, + "latency_ms": 13.353, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-22fa292f8766", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21367, + "latency_ms": 14.952, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-23339dc6c7ac", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36271, + "latency_ms": 12.337, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-254d8a38d40a", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 26995, + "latency_ms": 27.271, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-2bdf658143b0", + "path_leak": true, + "required_files": [ + "scripts/official-brand-publication.ts" + ], + "found": [], + "missed": [ + "scripts/official-brand-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26217, + "latency_ms": 14.455, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-3b759f36734b", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "found": [ + "scripts/reconcile.ts" + ], + "missed": [ + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 49463, + "latency_ms": 12.882, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-3c87dcc8228f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 66382, + "latency_ms": 14.842, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-3f67cd6936fb", + "path_leak": false, + "required_files": [ + "test/official-brand-sitemap.test.ts" + ], + "found": [], + "missed": [ + "test/official-brand-sitemap.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19886, + "latency_ms": 15.347, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-3fa2c53fa804", + "path_leak": false, + "required_files": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 52206, + "latency_ms": 14.872, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-40d2e4ea219e", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20279, + "latency_ms": 16.517, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-42eba2b71c9f", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40163, + "latency_ms": 14.533, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-52a00772a842", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26209, + "latency_ms": 15.416, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-54927ef9d7e2", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 11377, + "latency_ms": 17.876, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-5c62a39276da", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 36849, + "latency_ms": 15.726, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-6c2b66ff7837", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39039, + "latency_ms": 15.821, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-71d860be5034", + "path_leak": false, + "required_files": [ + "test/manual-publication-gate.test.ts" + ], + "found": [], + "missed": [ + "test/manual-publication-gate.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13728, + "latency_ms": 12.688, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-755dcb7a8906", + "path_leak": true, + "required_files": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 61473, + "latency_ms": 14.674, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-796f6b736144", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "found": [ + "scripts/review-bundles.ts" + ], + "missed": [ + "scripts/reconcile.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 46940, + "latency_ms": 16.748, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-880bc84a1840", + "path_leak": false, + "required_files": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "found": [], + "missed": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 176705, + "latency_ms": 15.644, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-88583294807f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51254, + "latency_ms": 16.521, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-8d4a8954b40e", + "path_leak": true, + "required_files": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "test/api.worker.test.ts" + ], + "missed": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 57760, + "latency_ms": 13.413, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-970e6366ccb9", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 17363, + "latency_ms": 24.927, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-985886cf2935", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 191083, + "latency_ms": 13.883, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-9b8659e2671b", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 51662, + "latency_ms": 12.576, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-a580a089b8cc", + "path_leak": true, + "required_files": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts" + ], + "missed": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 10, + "tokens_delivered": 27773, + "latency_ms": 14.1, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-b603f405353b", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26359, + "latency_ms": 15.709, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-b7e0001c5df7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27265, + "latency_ms": 15.311, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-be2529dd2357", + "path_leak": true, + "required_files": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "found": [ + "scripts/review-bundles.ts" + ], + "missed": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 43785, + "latency_ms": 16.271, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-c07e964dd0d2", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 201348, + "latency_ms": 13.432, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-c1ab8fe455d4", + "path_leak": true, + "required_files": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25632, + "latency_ms": 14.078, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-c634f70476c4", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32441, + "latency_ms": 13.317, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-c725a3c50f27", + "path_leak": false, + "required_files": [ + "src/App.tsx" + ], + "found": [ + "src/App.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 53078, + "latency_ms": 15.152, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-cadcf1bd67cf", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53150, + "latency_ms": 14.161, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-cb4efe626dc5", + "path_leak": false, + "required_files": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 180990, + "latency_ms": 31.124, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-d131feb0b34a", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33820, + "latency_ms": 13.235, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-d2003fb0158f", + "path_leak": true, + "required_files": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "found": [], + "missed": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49602, + "latency_ms": 12.97, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-d3b0639e3065", + "path_leak": false, + "required_files": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37735, + "latency_ms": 13.981, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-d3e9f665abd0", + "path_leak": false, + "required_files": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [], + "missed": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37975, + "latency_ms": 12.754, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-df42c030ac4d", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 35776, + "latency_ms": 12.765, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-e9b06828e033", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 46805, + "latency_ms": 12.989, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-e9dff924a117", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "found": [ + "test/domain.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 180693, + "latency_ms": 14.182, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-ee7d155b7b9e", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17672, + "latency_ms": 16.84, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-f4da150e6ba9", + "path_leak": true, + "required_files": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17339, + "latency_ms": 15.93, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-f9a264cd2de7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48848, + "latency_ms": 17.035, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-fda927717fdd", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 48561, + "latency_ms": 16.669, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-02bae53a54b2", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 109279, + "latency_ms": 25.046, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-12524e50831f", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 113847, + "latency_ms": 24.349, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-14a6c420d60c", + "path_leak": true, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 146146, + "latency_ms": 33.387, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-183e0a12fb52", + "path_leak": true, + "required_files": [ + "scripts/machine-label.ts" + ], + "found": [], + "missed": [ + "scripts/machine-label.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 243078, + "latency_ms": 35.85, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-196a41597b98", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "shared/nutrition.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.8, + "precision_at_10": 0.4, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 61735, + "latency_ms": 24.623, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-1a649b8e8091", + "path_leak": false, + "required_files": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.4, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.125, + "first_hit_rank": 1, + "tokens_delivered": 108324, + "latency_ms": 23.327, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-1af57b34dd19", + "path_leak": true, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 242323, + "latency_ms": 28.919, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-1e4d6787f0ec", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 188187, + "latency_ms": 29.43, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-22fa292f8766", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 111543, + "latency_ms": 30.997, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-23339dc6c7ac", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.8, + "precision_at_10": 0.4, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 1, + "tokens_delivered": 125200, + "latency_ms": 26.288, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-254d8a38d40a", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 145873, + "latency_ms": 26.083, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-2bdf658143b0", + "path_leak": true, + "required_files": [ + "scripts/official-brand-publication.ts" + ], + "found": [], + "missed": [ + "scripts/official-brand-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 242323, + "latency_ms": 31.513, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-3b759f36734b", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "found": [ + "scripts/reconcile.ts" + ], + "missed": [ + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 240039, + "latency_ms": 27.539, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-3c87dcc8228f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 242323, + "latency_ms": 26.407, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-3f67cd6936fb", + "path_leak": false, + "required_files": [ + "test/official-brand-sitemap.test.ts" + ], + "found": [], + "missed": [ + "test/official-brand-sitemap.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 242323, + "latency_ms": 30.224, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-3fa2c53fa804", + "path_leak": false, + "required_files": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 243078, + "latency_ms": 35.353, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-40d2e4ea219e", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 112625, + "latency_ms": 26.89, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-42eba2b71c9f", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 232569, + "latency_ms": 39.609, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-52a00772a842", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 59582, + "latency_ms": 23.357, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-54927ef9d7e2", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 255550, + "latency_ms": 28.999, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-5c62a39276da", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 61748, + "latency_ms": 22.891, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-6c2b66ff7837", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 241591, + "latency_ms": 36.369, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-71d860be5034", + "path_leak": false, + "required_files": [ + "test/manual-publication-gate.test.ts" + ], + "found": [], + "missed": [ + "test/manual-publication-gate.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 240039, + "latency_ms": 26.106, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-755dcb7a8906", + "path_leak": true, + "required_files": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/publication.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 104763, + "latency_ms": 21.586, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-796f6b736144", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "found": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "missed": [], + "recall_at_5": 0.6, + "precision_at_5": 0.6, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 1, + "tokens_delivered": 139291, + "latency_ms": 26.256, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-880bc84a1840", + "path_leak": false, + "required_files": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "found": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 105620, + "latency_ms": 20.627, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-88583294807f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 241591, + "latency_ms": 30.21, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-8d4a8954b40e", + "path_leak": true, + "required_files": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "shared/classification.ts", + "shared/metrics.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 105144, + "latency_ms": 23.435, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-970e6366ccb9", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 124661, + "latency_ms": 28.469, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-985886cf2935", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "shared/types.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 61708, + "latency_ms": 24.799, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-9b8659e2671b", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/robotoff.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 105882, + "latency_ms": 22.223, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-a580a089b8cc", + "path_leak": true, + "required_files": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "migrations/0007_review_queue_indexes.sql", + "shared/gtin.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 1, + "tokens_delivered": 109431, + "latency_ms": 51.591, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-b603f405353b", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/robotoff-ingredients-api.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 145983, + "latency_ms": 25.312, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-b7e0001c5df7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 240039, + "latency_ms": 25.164, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-be2529dd2357", + "path_leak": true, + "required_files": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "found": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 240119, + "latency_ms": 24.753, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-c07e964dd0d2", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 109089, + "latency_ms": 22.658, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-c1ab8fe455d4", + "path_leak": true, + "required_files": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-ingredients-api.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 189454, + "latency_ms": 25.366, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-c634f70476c4", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 232984, + "latency_ms": 32.304, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-c725a3c50f27", + "path_leak": false, + "required_files": [ + "src/App.tsx" + ], + "found": [ + "src/App.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 240159, + "latency_ms": 30.391, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-cadcf1bd67cf", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 233369, + "latency_ms": 27.551, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-cb4efe626dc5", + "path_leak": false, + "required_files": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/review-bundles.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 130024, + "latency_ms": 29.462, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-d131feb0b34a", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 240039, + "latency_ms": 33.604, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-d2003fb0158f", + "path_leak": true, + "required_files": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "found": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 104923, + "latency_ms": 24.392, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-d3b0639e3065", + "path_leak": false, + "required_files": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 140225, + "latency_ms": 26.877, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-d3e9f665abd0", + "path_leak": false, + "required_files": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "shared/nutrition.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 62240, + "latency_ms": 23.481, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-df42c030ac4d", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 146974, + "latency_ms": 31.649, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-e9b06828e033", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 55044, + "latency_ms": 27.115, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-e9dff924a117", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "test/domain.test.ts" + ], + "missed": [ + "shared/ingredients.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 50889, + "latency_ms": 30.969, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-ee7d155b7b9e", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 242323, + "latency_ms": 32.384, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-f4da150e6ba9", + "path_leak": true, + "required_files": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 233497, + "latency_ms": 28.946, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-f9a264cd2de7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 240039, + "latency_ms": 48.553, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-fda927717fdd", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 140313, + "latency_ms": 39.182, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-02bae53a54b2", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts", + "vitest.worker.config.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3374, + "latency_ms": 1340.388, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-12524e50831f", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 3396, + "latency_ms": 1404.198, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-14a6c420d60c", + "path_leak": true, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3959, + "latency_ms": 1407.058, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-183e0a12fb52", + "path_leak": true, + "required_files": [ + "scripts/machine-label.ts" + ], + "found": [ + "scripts/machine-label.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3935, + "latency_ms": 2219.926, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-196a41597b98", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "worker/catalog.ts" + ], + "missed": [ + "test/api.worker.test.ts", + "test/domain.test.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 5, + "tokens_delivered": 3841, + "latency_ms": 1181.576, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-1a649b8e8091", + "path_leak": false, + "required_files": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts" + ], + "missed": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0.125, + "precision_at_5": 0.2, + "recall_at_10": 0.375, + "precision_at_10": 0.3, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.125, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3788, + "latency_ms": 1463.676, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-1af57b34dd19", + "path_leak": true, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "found": [ + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 4422, + "latency_ms": 2314.255, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-1e4d6787f0ec", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3615, + "latency_ms": 1751.221, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-22fa292f8766", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3549, + "latency_ms": 1427.607, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-23339dc6c7ac", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.2308, + "recall_at_1000_tokens": 0.4, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 1, + "tokens_delivered": 3549, + "latency_ms": 1455.395, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-254d8a38d40a", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3523, + "latency_ms": 1515.103, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-2bdf658143b0", + "path_leak": true, + "required_files": [ + "scripts/official-brand-publication.ts" + ], + "found": [ + "scripts/official-brand-publication.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3548, + "latency_ms": 2257.36, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-3b759f36734b", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "found": [ + "scripts/reconcile.ts" + ], + "missed": [ + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3968, + "latency_ms": 2564.859, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-3c87dcc8228f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 3765, + "latency_ms": 2361.331, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-3f67cd6936fb", + "path_leak": false, + "required_files": [ + "test/official-brand-sitemap.test.ts" + ], + "found": [], + "missed": [ + "test/official-brand-sitemap.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3383, + "latency_ms": 2472.696, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-3fa2c53fa804", + "path_leak": false, + "required_files": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3462, + "latency_ms": 2369.746, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-40d2e4ea219e", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3480, + "latency_ms": 1556.873, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-42eba2b71c9f", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 4013, + "latency_ms": 2007.056, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-52a00772a842", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3690, + "latency_ms": 1399.939, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-54927ef9d7e2", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3935, + "latency_ms": 2292.176, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-5c62a39276da", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3478, + "latency_ms": 1183.486, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-6c2b66ff7837", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3614, + "latency_ms": 2336.727, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-71d860be5034", + "path_leak": false, + "required_files": [ + "test/manual-publication-gate.test.ts" + ], + "found": [], + "missed": [ + "test/manual-publication-gate.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3485, + "latency_ms": 2152.081, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-755dcb7a8906", + "path_leak": true, + "required_files": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/publication.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 3652, + "latency_ms": 1410.245, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-796f6b736144", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "found": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "worker/reviews.ts" + ], + "missed": [ + "test/api.worker.test.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.6, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 5, + "tokens_delivered": 4200, + "latency_ms": 1392.28, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-880bc84a1840", + "path_leak": false, + "required_files": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "found": [ + "worker/catalog.ts" + ], + "missed": [ + "test/api.worker.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 3498, + "latency_ms": 1329.951, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-88583294807f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 3169, + "latency_ms": 2455.28, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-8d4a8954b40e", + "path_leak": true, + "required_files": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "worker/catalog.ts" + ], + "missed": [ + "test/api.worker.test.ts", + "test/domain.test.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.8, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.2667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3936, + "latency_ms": 1393.286, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-970e6366ccb9", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 4286, + "latency_ms": 1387.654, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-985886cf2935", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 4, + "tokens_delivered": 3847, + "latency_ms": 1288.104, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-9b8659e2671b", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3653, + "latency_ms": 2031.054, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-a580a089b8cc", + "path_leak": true, + "required_files": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/reconcile.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "shared/gtin.ts" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.2308, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3144, + "latency_ms": 1585.709, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-b603f405353b", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3541, + "latency_ms": 1569.212, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-b7e0001c5df7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [ + "scripts/guarded-publication.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 3877, + "latency_ms": 2370.753, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-be2529dd2357", + "path_leak": true, + "required_files": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "found": [ + "scripts/guarded-publication.ts", + "scripts/review-bundles.ts" + ], + "missed": [ + "scripts/reconcile.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2222, + "recall_at_20": 0.6667, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 3771, + "latency_ms": 2189.637, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-c07e964dd0d2", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 9, + "tokens_delivered": 3843, + "latency_ms": 1510.754, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-c1ab8fe455d4", + "path_leak": true, + "required_files": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.75, + "precision_at_5": 0.6, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 3970, + "latency_ms": 1806.216, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-c634f70476c4", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4200, + "latency_ms": 2147.093, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-c725a3c50f27", + "path_leak": false, + "required_files": [ + "src/App.tsx" + ], + "found": [], + "missed": [ + "src/App.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3652, + "latency_ms": 2404.434, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-cadcf1bd67cf", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3739, + "latency_ms": 2166.664, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-cb4efe626dc5", + "path_leak": false, + "required_files": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/review-bundles.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2880, + "latency_ms": 1670.04, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-d131feb0b34a", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [ + "scripts/guarded-publication.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 3762, + "latency_ms": 2246.766, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-d2003fb0158f", + "path_leak": true, + "required_files": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "found": [ + "worker/coverage.ts" + ], + "missed": [ + "test/api.worker.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3311, + "latency_ms": 1474.673, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-d3b0639e3065", + "path_leak": false, + "required_files": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3291, + "latency_ms": 1839.851, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-d3e9f665abd0", + "path_leak": false, + "required_files": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "shared/nutrition.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "test/api.worker.test.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 3880, + "latency_ms": 2009.585, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-df42c030ac4d", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3943, + "latency_ms": 1609.977, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-e9b06828e033", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4757, + "latency_ms": 1325.099, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-e9dff924a117", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts" + ], + "missed": [ + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3486, + "latency_ms": 1327.279, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-ee7d155b7b9e", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4126, + "latency_ms": 2431.168, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-f4da150e6ba9", + "path_leak": true, + "required_files": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3520, + "latency_ms": 2333.33, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-f9a264cd2de7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [ + "scripts/guarded-publication.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3622, + "latency_ms": 2534.338, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "protein-index-fda927717fdd", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 2891, + "latency_ms": 1614.751, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-02bae53a54b2", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 582, + "latency_ms": 632.323, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-12524e50831f", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 553, + "latency_ms": 523.442, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-14a6c420d60c", + "path_leak": true, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 742, + "latency_ms": 600.37, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-183e0a12fb52", + "path_leak": true, + "required_files": [ + "scripts/machine-label.ts" + ], + "found": [ + "scripts/machine-label.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 583, + "latency_ms": 811.955, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-196a41597b98", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "test/api.worker.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.3333, + "recall_at_10": 0.2, + "precision_at_10": 0.3333, + "recall_at_20": 0.2, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 498, + "latency_ms": 673.045, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-1a649b8e8091", + "path_leak": false, + "required_files": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 1, + "recall_at_10": 0.25, + "precision_at_10": 1, + "recall_at_20": 0.25, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 747, + "latency_ms": 652.456, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-1af57b34dd19", + "path_leak": true, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "found": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "missed": [ + "config/official-brand-sources.json", + "scripts/official-brand-publication.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 786, + "latency_ms": 903.371, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-1e4d6787f0ec", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 612, + "latency_ms": 778.816, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-22fa292f8766", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 397, + "latency_ms": 565.336, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-23339dc6c7ac", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 503, + "latency_ms": 626.085, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-254d8a38d40a", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 452, + "latency_ms": 611.954, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-2bdf658143b0", + "path_leak": true, + "required_files": [ + "scripts/official-brand-publication.ts" + ], + "found": [ + "scripts/official-brand-publication.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 624, + "latency_ms": 960.691, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-3b759f36734b", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "found": [], + "missed": [ + "scripts/reconcile.ts", + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 680, + "latency_ms": 879.192, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-3c87dcc8228f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 572, + "latency_ms": 984.572, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-3f67cd6936fb", + "path_leak": false, + "required_files": [ + "test/official-brand-sitemap.test.ts" + ], + "found": [], + "missed": [ + "test/official-brand-sitemap.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1175, + "latency_ms": 1990.289, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-3fa2c53fa804", + "path_leak": false, + "required_files": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 632, + "latency_ms": 1026.745, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-40d2e4ea219e", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 733, + "latency_ms": 741.285, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-42eba2b71c9f", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 459, + "latency_ms": 926.009, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-52a00772a842", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 530, + "latency_ms": 681.751, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-54927ef9d7e2", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 523, + "latency_ms": 1042.561, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-5c62a39276da", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 635, + "latency_ms": 688.529, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-6c2b66ff7837", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 639, + "latency_ms": 1172.489, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-71d860be5034", + "path_leak": false, + "required_files": [ + "test/manual-publication-gate.test.ts" + ], + "found": [], + "missed": [ + "test/manual-publication-gate.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 560, + "latency_ms": 941.075, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-755dcb7a8906", + "path_leak": true, + "required_files": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/publication.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 521, + "latency_ms": 746.279, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-796f6b736144", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "worker/reviews.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.5, + "recall_at_10": 0.2, + "precision_at_10": 0.5, + "recall_at_20": 0.2, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 617, + "latency_ms": 523.578, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-880bc84a1840", + "path_leak": false, + "required_files": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "found": [], + "missed": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 615, + "latency_ms": 695.132, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-88583294807f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 732, + "latency_ms": 600.921, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-8d4a8954b40e", + "path_leak": true, + "required_files": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "test/api.worker.test.ts", + "test/domain.test.ts" + ], + "missed": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.6667, + "recall_at_10": 0.3333, + "precision_at_10": 0.6667, + "recall_at_20": 0.3333, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 587, + "latency_ms": 419.081, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-970e6366ccb9", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 466, + "latency_ms": 572.651, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-985886cf2935", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "shared/types.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 490, + "latency_ms": 552.813, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-9b8659e2671b", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 386, + "latency_ms": 471.055, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-a580a089b8cc", + "path_leak": true, + "required_files": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 486, + "latency_ms": 425.198, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-b603f405353b", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 536, + "latency_ms": 599.577, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-b7e0001c5df7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 479, + "latency_ms": 667.109, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-be2529dd2357", + "path_leak": true, + "required_files": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "found": [ + "scripts/review-bundles.ts" + ], + "missed": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 591, + "latency_ms": 680.118, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-c07e964dd0d2", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.25, + "recall_at_10": 0.3333, + "precision_at_10": 0.25, + "recall_at_20": 0.3333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 453, + "latency_ms": 546.803, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-c1ab8fe455d4", + "path_leak": true, + "required_files": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 628, + "latency_ms": 599.314, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-c634f70476c4", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 681, + "latency_ms": 694.451, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-c725a3c50f27", + "path_leak": false, + "required_files": [ + "src/App.tsx" + ], + "found": [ + "src/App.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 660, + "latency_ms": 722.649, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-cadcf1bd67cf", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 495, + "latency_ms": 697.683, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-cb4efe626dc5", + "path_leak": false, + "required_files": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 593, + "latency_ms": 546.115, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-d131feb0b34a", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 735, + "latency_ms": 736.578, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-d2003fb0158f", + "path_leak": true, + "required_files": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "found": [], + "missed": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 403, + "latency_ms": 564.186, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-d3b0639e3065", + "path_leak": false, + "required_files": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 607, + "latency_ms": 620.495, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-d3e9f665abd0", + "path_leak": false, + "required_files": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "shared/nutrition.ts" + ], + "missed": [ + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.5, + "recall_at_10": 0.25, + "precision_at_10": 0.5, + "recall_at_20": 0.25, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 558, + "latency_ms": 505.242, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-df42c030ac4d", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 751, + "latency_ms": 656.77, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-e9b06828e033", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 539, + "latency_ms": 608.556, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-e9dff924a117", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 460, + "latency_ms": 568.342, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-ee7d155b7b9e", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 694, + "latency_ms": 904.369, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-f4da150e6ba9", + "path_leak": true, + "required_files": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/decision-drift-audit.ts" + ], + "missed": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.3333, + "recall_at_10": 0.2, + "precision_at_10": 0.3333, + "recall_at_20": 0.2, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 917, + "latency_ms": 936.781, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-f9a264cd2de7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [ + "scripts/guarded-publication.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 532, + "latency_ms": 963.992, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "protein-index-fda927717fdd", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 569, + "latency_ms": 771.702, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-02bae53a54b2", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4790, + "latency_ms": 10312.475, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-12524e50831f", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2876, + "latency_ms": 4964.995, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-14a6c420d60c", + "path_leak": true, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5238, + "latency_ms": 12555.516, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-183e0a12fb52", + "path_leak": true, + "required_files": [ + "scripts/machine-label.ts" + ], + "found": [], + "missed": [ + "scripts/machine-label.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5237, + "latency_ms": 23663.045, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-196a41597b98", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5146, + "latency_ms": 114055.361, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-1a649b8e8091", + "path_leak": false, + "required_files": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.375, + "precision_at_10": 0.3, + "recall_at_20": 0.375, + "precision_at_20": 0.2308, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.375, + "recall_at_16000_tokens": 0.375, + "first_hit_rank": 7, + "tokens_delivered": 5180, + "latency_ms": 22390.292, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-1af57b34dd19", + "path_leak": true, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4641, + "latency_ms": 66135.66, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-1e4d6787f0ec", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4466, + "latency_ms": 117093.748, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-22fa292f8766", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4760, + "latency_ms": 15656.4, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-23339dc6c7ac", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts", + "worker/catalog.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.25, + "recall_at_20": 0.4, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 5, + "tokens_delivered": 3644, + "latency_ms": 221666.231, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-254d8a38d40a", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1855.415, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-2bdf658143b0", + "path_leak": true, + "required_files": [ + "scripts/official-brand-publication.ts" + ], + "found": [], + "missed": [ + "scripts/official-brand-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 575.901, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-3b759f36734b", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "found": [], + "missed": [ + "scripts/reconcile.ts", + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 762.421, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-3c87dcc8228f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 690.646, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-3f67cd6936fb", + "path_leak": false, + "required_files": [ + "test/official-brand-sitemap.test.ts" + ], + "found": [], + "missed": [ + "test/official-brand-sitemap.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 657.315, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-3fa2c53fa804", + "path_leak": false, + "required_files": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 614.212, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-40d2e4ea219e", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 697.617, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-42eba2b71c9f", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 671.08, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-52a00772a842", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 731.714, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-54927ef9d7e2", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 731.644, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-5c62a39276da", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1004.146, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-6c2b66ff7837", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 644.811, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-71d860be5034", + "path_leak": false, + "required_files": [ + "test/manual-publication-gate.test.ts" + ], + "found": [], + "missed": [ + "test/manual-publication-gate.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 637.442, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-755dcb7a8906", + "path_leak": true, + "required_files": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 775.92, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-796f6b736144", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "found": [], + "missed": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 766.7, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-880bc84a1840", + "path_leak": false, + "required_files": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "found": [], + "missed": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 835.727, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-88583294807f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 718.312, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-8d4a8954b40e", + "path_leak": true, + "required_files": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [], + "missed": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 776.652, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-970e6366ccb9", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 876.299, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-985886cf2935", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 839.144, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-9b8659e2671b", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 795.063, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-a580a089b8cc", + "path_leak": true, + "required_files": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 601.782, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-b603f405353b", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 786.133, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-b7e0001c5df7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 717.255, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-be2529dd2357", + "path_leak": true, + "required_files": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 718.269, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-c07e964dd0d2", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 975.938, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-c1ab8fe455d4", + "path_leak": true, + "required_files": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 796.913, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-c634f70476c4", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 990.737, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-c725a3c50f27", + "path_leak": false, + "required_files": [ + "src/App.tsx" + ], + "found": [], + "missed": [ + "src/App.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 885.62, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-cadcf1bd67cf", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 601.416, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-cb4efe626dc5", + "path_leak": false, + "required_files": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 641.893, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-d131feb0b34a", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 556.22, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-d2003fb0158f", + "path_leak": true, + "required_files": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "found": [], + "missed": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 608.824, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-d3b0639e3065", + "path_leak": false, + "required_files": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 572.378, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-d3e9f665abd0", + "path_leak": false, + "required_files": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [], + "missed": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 574.22, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-df42c030ac4d", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 605.127, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-e9b06828e033", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 538.561, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-e9dff924a117", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 602.139, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-ee7d155b7b9e", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 704.454, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-f4da150e6ba9", + "path_leak": true, + "required_files": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 629.952, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-f9a264cd2de7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 668.325, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "protein-index-fda927717fdd", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 661.014, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/P-site-health.json b/benchmarks/context-retrieval/results/P-site-health.json new file mode 100644 index 00000000..d061a59f --- /dev/null +++ b/benchmarks/context-retrieval/results/P-site-health.json @@ -0,0 +1,12966 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "site-health", + "head": "c4ea582e0a8371c566e3e832485fc33da1a318c6" + }, + "tier": "small", + "tier_code_files": 69, + "corpus_counts": { + "cases": 88, + "path_leak": 45, + "baseline_blind": 43, + "multi_file": 49 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.3006, + "control": 0.0114, + "reason": null + }, + "needs_raw_payload_check": [ + { + "provider_id": "cocoindex-code", + "why": "scored exactly zero — read the raw payload before believing it" + } + ], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "complete mobile preview pane platform split", + "revision": "cde2e33f06097c1ee56dd6b7f61f9ea135b7b34b", + "returned": [ + "apps/mobile/src/lib/use-voice-input.ts", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/src/components/preview-pane.ts", + "packages/bridge/tsconfig.json", + "apps/mobile/src/components/ui.tsx" + ] + }, + { + "provider_id": "random-files", + "query": "omit unavailable public product links", + "revision": "7d2d155a89368326a4346e874f9b9a866e922151", + "returned": [ + "openspec/changes/archive/2026-07-13-fleet-marketing-control-plane/.openspec.yaml", + "fleet-ops/services/content-factory/scripts/render-content-package.js", + "fleet-ops/services/reel-pipeline/test/fixtures/saas-maker-improvement.json", + "fleet-ops/services/drank/data/fleet-dr.json", + "fleet-ops/apps/mobile-cockpit/openspec/config.yaml" + ] + }, + { + "provider_id": "random-files", + "query": "make native simulator workflow functional", + "revision": "eff69187f6b7d044a8ae946b8ffddc60b1542eff", + "returned": [ + "packages/bridge/tsconfig.build.json", + "apps/mobile/test/layout.test.ts", + "apps/mobile/src/app/index.tsx", + "apps/mobile/test/ios-scene-lifecycle-plugin.test.js", + "apps/mobile/modules/cockpit-voice/expo-module.config.json" + ] + }, + { + "provider_id": "churn-ranked", + "query": "show Search reporting period", + "revision": "4b357538b4bd04e389d89b138127dd036152c466", + "returned": [ + "package.json", + "foundry/ops/config/projects.json", + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/test/marketing-program.test.mjs" + ] + }, + { + "provider_id": "churn-ranked", + "query": "expose provider console links", + "revision": "66d72c5c3b4060fbd32304aad8d1023fd704deb8", + "returned": [ + "package.json", + "foundry/ops/config/projects.json", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json" + ] + }, + { + "provider_id": "semble", + "query": "show Search reporting period", + "revision": "4b357538b4bd04e389d89b138127dd036152c466", + "returned": [ + "foundry/ops/lib/search-console.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/ops/lib/visibility-outcome-store.mjs" + ] + }, + { + "provider_id": "semble", + "query": "Fix data page lint errors", + "revision": "d2b439351a29fcbecb257f7b1ac4146f5a040a4b", + "returned": [ + "app/sitemap.ts", + "scripts/update-global-dr.mjs", + "lib/foundry-monitoring.ts", + "functions/api/dr.ts", + "lib/useTrackedDomains.ts" + ] + }, + { + "provider_id": "jcodemunch", + "query": "expose latest learning (#18)", + "revision": "ad33fe016b0e905af2f3534fdc254bec17ca1ba9", + "returned": [ + "foundry/apps/public-directory/src/styles/globals.css" + ] + }, + { + "provider_id": "jcodemunch", + "query": "monitor stale field evidence", + "revision": "b43186e6c2d64afd76a5acbe61d9f383709fe921", + "returned": [ + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ] + }, + { + "provider_id": "jcodemunch", + "query": "cover all ten roots in Search Console (#197)", + "revision": "a75b12884352a32b103628938ed66cb8a727c24b", + "returned": [ + "foundry/ops/test/root-search-query-contract.test.mjs", + "foundry/ops/lib/search-console.mjs" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 88, + "mean_recall_at_5": 0.0114, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0152, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0492, + "full_recall_at_20": 0.0227, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0114, + "mean_recall_at_16000_tokens": 0.0331, + "mean_precision_at_10": 0.0034, + "zero_hit_rate": 0.8977, + "median_tokens_delivered": 28531, + "median_latency_ms": 17.9235, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 88, + "mean_recall_at_5": 0.0114, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0152, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0492, + "full_recall_at_20": 0.0227, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0114, + "mean_recall_at_16000_tokens": 0.0331, + "mean_precision_at_10": 0.0034, + "zero_hit_rate": 0.8977, + "median_tokens_delivered": 28531, + "median_latency_ms": 17.9235, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 45, + "mean_recall_at_5": 0.0111, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0185, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0556, + "full_recall_at_20": 0.0222, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0111, + "mean_recall_at_16000_tokens": 0.0241, + "mean_precision_at_10": 0.0044, + "zero_hit_rate": 0.8667, + "median_tokens_delivered": 33112, + "median_latency_ms": 17.559, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 43, + "mean_recall_at_5": 0.0116, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0116, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0426, + "full_recall_at_20": 0.0233, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0116, + "mean_recall_at_16000_tokens": 0.0426, + "mean_precision_at_10": 0.0023, + "zero_hit_rate": 0.9302, + "median_tokens_delivered": 27834, + "median_latency_ms": 18.27, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 88 + }, + "resources": { + "baseline_rss_mb": 16835.6, + "peak_rss_mb": 17149.3, + "delta_rss_mb": 313.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 88, + "mean_recall_at_5": 0.1315, + "full_recall_at_5": 0.0795, + "mean_recall_at_10": 0.1947, + "full_recall_at_10": 0.1136, + "mean_recall_at_20": 0.2466, + "full_recall_at_20": 0.1705, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0038, + "mean_recall_at_16000_tokens": 0.068, + "mean_precision_at_10": 0.0466, + "zero_hit_rate": 0.6591, + "median_tokens_delivered": 139176, + "median_latency_ms": 85.154, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 88, + "mean_recall_at_5": 0.1315, + "full_recall_at_5": 0.0795, + "mean_recall_at_10": 0.1947, + "full_recall_at_10": 0.1136, + "mean_recall_at_20": 0.2466, + "full_recall_at_20": 0.1705, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0038, + "mean_recall_at_16000_tokens": 0.068, + "mean_precision_at_10": 0.0466, + "zero_hit_rate": 0.6591, + "median_tokens_delivered": 139176, + "median_latency_ms": 85.154, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 45, + "mean_recall_at_5": 0.0756, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.133, + "full_recall_at_10": 0.0222, + "mean_recall_at_20": 0.1938, + "full_recall_at_20": 0.0889, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0074, + "mean_recall_at_16000_tokens": 0.0367, + "mean_precision_at_10": 0.0489, + "zero_hit_rate": 0.6667, + "median_tokens_delivered": 68366, + "median_latency_ms": 82.679, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 43, + "mean_recall_at_5": 0.1899, + "full_recall_at_5": 0.1628, + "mean_recall_at_10": 0.2593, + "full_recall_at_10": 0.2093, + "mean_recall_at_20": 0.3019, + "full_recall_at_20": 0.2558, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1008, + "mean_precision_at_10": 0.0442, + "zero_hit_rate": 0.6512, + "median_tokens_delivered": 142313, + "median_latency_ms": 87.441, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 88 + }, + "resources": { + "baseline_rss_mb": 16521.3, + "peak_rss_mb": 17482.3, + "delta_rss_mb": 960.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 88, + "mean_recall_at_5": 0.204, + "full_recall_at_5": 0.125, + "mean_recall_at_10": 0.2606, + "full_recall_at_10": 0.1591, + "mean_recall_at_20": 0.3022, + "full_recall_at_20": 0.1818, + "mean_recall_at_1000_tokens": 0.1832, + "mean_recall_at_4000_tokens": 0.3006, + "mean_recall_at_16000_tokens": 0.3022, + "mean_precision_at_10": 0.0581, + "zero_hit_rate": 0.5455, + "median_tokens_delivered": 3740, + "median_latency_ms": 4521.88, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 88, + "mean_recall_at_5": 0.204, + "full_recall_at_5": 0.125, + "mean_recall_at_10": 0.2606, + "full_recall_at_10": 0.1591, + "mean_recall_at_20": 0.3022, + "full_recall_at_20": 0.1818, + "mean_recall_at_1000_tokens": 0.1832, + "mean_recall_at_4000_tokens": 0.3006, + "mean_recall_at_16000_tokens": 0.3022, + "mean_precision_at_10": 0.0581, + "zero_hit_rate": 0.5455, + "median_tokens_delivered": 3740, + "median_latency_ms": 4521.88, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 45, + "mean_recall_at_5": 0.2271, + "full_recall_at_5": 0.1333, + "mean_recall_at_10": 0.2785, + "full_recall_at_10": 0.1556, + "mean_recall_at_20": 0.2962, + "full_recall_at_20": 0.1556, + "mean_recall_at_1000_tokens": 0.2048, + "mean_recall_at_4000_tokens": 0.293, + "mean_recall_at_16000_tokens": 0.2962, + "mean_precision_at_10": 0.0669, + "zero_hit_rate": 0.5111, + "median_tokens_delivered": 3841, + "median_latency_ms": 4035.595, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 43, + "mean_recall_at_5": 0.1798, + "full_recall_at_5": 0.1163, + "mean_recall_at_10": 0.2419, + "full_recall_at_10": 0.1628, + "mean_recall_at_20": 0.3085, + "full_recall_at_20": 0.2093, + "mean_recall_at_1000_tokens": 0.1605, + "mean_recall_at_4000_tokens": 0.3085, + "mean_recall_at_16000_tokens": 0.3085, + "mean_precision_at_10": 0.0488, + "zero_hit_rate": 0.5814, + "median_tokens_delivered": 3715, + "median_latency_ms": 5334.548, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 88 + }, + "resources": { + "baseline_rss_mb": 17381.4, + "peak_rss_mb": 22683.2, + "delta_rss_mb": 5301.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "jcodemunch", + "summary": { + "all": { + "cases": 88, + "mean_recall_at_5": 0.1309, + "full_recall_at_5": 0.0795, + "mean_recall_at_10": 0.1309, + "full_recall_at_10": 0.0795, + "mean_recall_at_20": 0.1309, + "full_recall_at_20": 0.0795, + "mean_recall_at_1000_tokens": 0.1309, + "mean_recall_at_4000_tokens": 0.1309, + "mean_recall_at_16000_tokens": 0.1309, + "mean_precision_at_10": 0.1686, + "zero_hit_rate": 0.7727, + "median_tokens_delivered": 553.5, + "median_latency_ms": 2538.2195, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 11 + }, + "baseline_missed": { + "cases": 88, + "mean_recall_at_5": 0.1309, + "full_recall_at_5": 0.0795, + "mean_recall_at_10": 0.1309, + "full_recall_at_10": 0.0795, + "mean_recall_at_20": 0.1309, + "full_recall_at_20": 0.0795, + "mean_recall_at_1000_tokens": 0.1309, + "mean_recall_at_4000_tokens": 0.1309, + "mean_recall_at_16000_tokens": 0.1309, + "mean_precision_at_10": 0.1686, + "zero_hit_rate": 0.7727, + "median_tokens_delivered": 553.5, + "median_latency_ms": 2538.2195, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 11 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 45, + "mean_recall_at_5": 0.1356, + "full_recall_at_5": 0.0889, + "mean_recall_at_10": 0.1356, + "full_recall_at_10": 0.0889, + "mean_recall_at_20": 0.1356, + "full_recall_at_20": 0.0889, + "mean_recall_at_1000_tokens": 0.1356, + "mean_recall_at_4000_tokens": 0.1356, + "mean_recall_at_16000_tokens": 0.1356, + "mean_precision_at_10": 0.1519, + "zero_hit_rate": 0.7778, + "median_tokens_delivered": 558, + "median_latency_ms": 2296.94, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 3 + }, + "no_path_leak": { + "cases": 43, + "mean_recall_at_5": 0.126, + "full_recall_at_5": 0.0698, + "mean_recall_at_10": 0.126, + "full_recall_at_10": 0.0698, + "mean_recall_at_20": 0.126, + "full_recall_at_20": 0.0698, + "mean_recall_at_1000_tokens": 0.126, + "mean_recall_at_4000_tokens": 0.126, + "mean_recall_at_16000_tokens": 0.126, + "mean_precision_at_10": 0.186, + "zero_hit_rate": 0.7674, + "median_tokens_delivered": 553, + "median_latency_ms": 2733.561, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 8 + } + }, + "outcomes": { + "answered": 77, + "indexed-but-returned-nothing": 11 + }, + "resources": { + "baseline_rss_mb": 21440.9, + "peak_rss_mb": 25558.8, + "delta_rss_mb": 4117.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "cocoindex-code", + "summary": { + "all": { + "cases": 88, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 2607, + "median_latency_ms": 528.991, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 88, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 2607, + "median_latency_ms": 528.991, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 45, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 2607, + "median_latency_ms": 523.25, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 43, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 2607, + "median_latency_ms": 554.817, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 88 + }, + "resources": { + "baseline_rss_mb": 20774.9, + "peak_rss_mb": 21963.7, + "delta_rss_mb": 1188.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "site-health-00d7bb046471", + "path_leak": false, + "required_files": [ + "apps/backend/config/projects.json" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23275, + "latency_ms": 41.05, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-010e4aebe75e", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 669466, + "latency_ms": 24.466, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-054409454be6", + "path_leak": false, + "required_files": [ + ".gitmodules" + ], + "found": [], + "missed": [ + ".gitmodules" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44685, + "latency_ms": 18.622, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-071417754558", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/next.config.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16899, + "latency_ms": 27.549, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-07ad25caddd4", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22403, + "latency_ms": 17.884, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-0ba9b5f5948b", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27851, + "latency_ms": 18.563, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-0baeb9010cb3", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62897, + "latency_ms": 28.894, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-132847b20d6d", + "path_leak": false, + "required_files": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17187, + "latency_ms": 20.921, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-18e656d8741b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74067, + "latency_ms": 16.378, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-19f05b318577", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18822, + "latency_ms": 15.433, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-1ca619fba9f2", + "path_leak": true, + "required_files": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 118893, + "latency_ms": 18.036, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-1d00d5a8b1af", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27834, + "latency_ms": 20.088, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-21839f193280", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [], + "missed": [ + "app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12713, + "latency_ms": 16.759, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-2451fc913f88", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34583, + "latency_ms": 18.683, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-2991f2b749b6", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28233, + "latency_ms": 18.03, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-2d81c1441e17", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20297, + "latency_ms": 22.121, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-2e7392f7dccd", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28206, + "latency_ms": 16.1, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-3224f6662580", + "path_leak": true, + "required_files": [ + "apps/mobile/src/components/preview-pane.d.ts", + "apps/mobile/src/components/preview-pane.ts" + ], + "found": [ + "apps/mobile/src/components/preview-pane.ts" + ], + "missed": [ + "apps/mobile/src/components/preview-pane.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 20739, + "latency_ms": 15.256, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-32e6fd1b5c87", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27802, + "latency_ms": 17.251, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-40426bd7a03a", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27089, + "latency_ms": 17.317, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-41ad25bba286", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39614, + "latency_ms": 21.291, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-464fe2d9d985", + "path_leak": false, + "required_files": [ + "biome.json" + ], + "found": [], + "missed": [ + "biome.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 97735, + "latency_ms": 14.628, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-4b3332684075", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28496, + "latency_ms": 27.362, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-4c00f66be11b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20816, + "latency_ms": 19.272, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-4efb5e0e9f6c", + "path_leak": false, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36743, + "latency_ms": 14.633, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-4f8e172c77c0", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23813, + "latency_ms": 17.387, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-58ce619558e4", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84109, + "latency_ms": 23.815, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5947cb9235a4", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13196, + "latency_ms": 19.493, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5add561d3a1c", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31271, + "latency_ms": 17.201, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5c274bfb8d8c", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47320, + "latency_ms": 14.75, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5c737aa349c4", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [ + "app/layout.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 13140, + "latency_ms": 15.084, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5e9dd3d3a183", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35626, + "latency_ms": 31.918, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5f057c9840e6", + "path_leak": true, + "required_files": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28196, + "latency_ms": 14.481, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5ff8c1d23d1a", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27122, + "latency_ms": 15.781, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-607ffead8d59", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40077, + "latency_ms": 17.364, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-6410c21e39fb", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41266, + "latency_ms": 15.512, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-6806c601cd49", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 76806, + "latency_ms": 12.929, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-6a61e9774eee", + "path_leak": true, + "required_files": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/design-workflow.mjs" + ], + "found": [ + "foundry/ops/scripts/design-workflow.mjs" + ], + "missed": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 13, + "tokens_delivered": 21229, + "latency_ms": 15.156, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-6e4082265380", + "path_leak": true, + "required_files": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28566, + "latency_ms": 17.597, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-70a328abccdb", + "path_leak": false, + "required_files": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "found": [], + "missed": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15368, + "latency_ms": 15.615, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-747d98b7face", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 61753, + "latency_ms": 17.963, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-756936a2820c", + "path_leak": false, + "required_files": [ + "scripts/prepare-node-pty.mjs" + ], + "found": [], + "missed": [ + "scripts/prepare-node-pty.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16437, + "latency_ms": 15.407, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-834566d04be7", + "path_leak": true, + "required_files": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "found": [ + "foundry/services/reel-pipeline/src/film-skills.js" + ], + "missed": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 42577, + "latency_ms": 18.695, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-864bb66e08c9", + "path_leak": false, + "required_files": [ + "foundry/ops/config/projects.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64399, + "latency_ms": 20.598, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-8c06ab66e57f", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22437, + "latency_ms": 18.701, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-90d2bee838d8", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33970, + "latency_ms": 19.435, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-93e088e6145f", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29219, + "latency_ms": 17.086, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-98062c56ade9", + "path_leak": true, + "required_files": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/project-catalog.test.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/project-catalog.test.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21450, + "latency_ms": 19.038, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-9afe0805eedb", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "found": [ + "foundry/packages/portfolio-project-strip/src/catalog.ts" + ], + "missed": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 33719, + "latency_ms": 35.971, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-9d823e16ea99", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/search-console.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/search-console.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 79649, + "latency_ms": 19.661, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-9e0367722215", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28146, + "latency_ms": 18.41, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-9f748e8bfd7a", + "path_leak": false, + "required_files": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts", + "packages/protocol/src/index.ts" + ], + "found": [ + "packages/protocol/src/index.ts" + ], + "missed": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 17, + "tokens_delivered": 17551, + "latency_ms": 14.39, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-a29095247805", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22657, + "latency_ms": 18.219, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-a67cc73f8c17", + "path_leak": true, + "required_files": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "found": [ + "fleet-ops/public/products.json" + ], + "missed": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 9, + "tokens_delivered": 20530, + "latency_ms": 16.561, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-a766a9fe6db6", + "path_leak": true, + "required_files": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "found": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 56960, + "latency_ms": 15.262, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-a7e020f5380c", + "path_leak": true, + "required_files": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 87173, + "latency_ms": 17.353, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-aa702f549db8", + "path_leak": true, + "required_files": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44921, + "latency_ms": 27.281, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-abf1e1c3dcf9", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93641, + "latency_ms": 20.686, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-ac2d8a08ccf7", + "path_leak": true, + "required_files": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25118, + "latency_ms": 14.802, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-ad8c488454cf", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/git-health.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/git-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32389, + "latency_ms": 14.026, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-b311494c1853", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34931, + "latency_ms": 21, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-b43186e6c2d6", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22315, + "latency_ms": 17.693, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-b460bbee477e", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/page.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/page.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 119820, + "latency_ms": 19.08, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-b963e53c1599", + "path_leak": false, + "required_files": [ + "foundry/ops/config/marketing-program.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/marketing-program.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22797, + "latency_ms": 18.27, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-b9a9b7fb722e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 108763, + "latency_ms": 17.766, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c055fb854a9c", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30638, + "latency_ms": 17.865, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c05c3a7fd02e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34548, + "latency_ms": 17.146, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c30d08ebb49f", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30735, + "latency_ms": 33.698, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c3cb6dd9d680", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13152, + "latency_ms": 19.373, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c4f7e2d22b0c", + "path_leak": true, + "required_files": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24733, + "latency_ms": 20.478, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c50c5280c323", + "path_leak": true, + "required_files": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12255, + "latency_ms": 34.657, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c99064f83b83", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 95525, + "latency_ms": 17.247, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-ca50c4d300dd", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33858, + "latency_ms": 17.413, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-d906a7364744", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33529, + "latency_ms": 17.073, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-e132f96c25ac", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15978, + "latency_ms": 17.607, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-e261ca6aa79e", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24720, + "latency_ms": 26.951, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-e288e502d891", + "path_leak": false, + "required_files": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "found": [ + "apps/mobile/src/app/index.tsx" + ], + "missed": [ + "packages/bridge/src/discover.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 14889, + "latency_ms": 15.295, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-e4a52d0adfb6", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 94875, + "latency_ms": 18.509, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-e7dec365bcf9", + "path_leak": false, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26039, + "latency_ms": 16.464, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-e8113343338f", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 88690, + "latency_ms": 17.237, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-eaaf7a67fc35", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42075, + "latency_ms": 18.838, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-efdb446d024f", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21596, + "latency_ms": 18.816, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-f157706d6ddc", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 65479, + "latency_ms": 20.205, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-f27818367be8", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25830, + "latency_ms": 21.049, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-f2b275130ace", + "path_leak": true, + "required_files": [ + "fleet-ops/config/marketing-program.json" + ], + "found": [], + "missed": [ + "fleet-ops/config/marketing-program.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21528, + "latency_ms": 17.559, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-f7557d39e6e6", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 98211, + "latency_ms": 21.295, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-f8b07be6f191", + "path_leak": true, + "required_files": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "found": [], + "missed": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13658, + "latency_ms": 16.186, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-fa0e527907ad", + "path_leak": true, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33112, + "latency_ms": 17.388, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-00d7bb046471", + "path_leak": false, + "required_files": [ + "apps/backend/config/projects.json" + ], + "found": [ + "apps/backend/config/projects.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 87102, + "latency_ms": 217.84, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-010e4aebe75e", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 184580, + "latency_ms": 89.664, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-054409454be6", + "path_leak": false, + "required_files": [ + ".gitmodules" + ], + "found": [], + "missed": [ + ".gitmodules" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 52509, + "latency_ms": 30.976, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-071417754558", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/next.config.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159680, + "latency_ms": 96.321, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-07ad25caddd4", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 57622, + "latency_ms": 63.848, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-0ba9b5f5948b", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159689, + "latency_ms": 107.13, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-0baeb9010cb3", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 141878, + "latency_ms": 85.999, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-132847b20d6d", + "path_leak": false, + "required_files": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "found": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 160128, + "latency_ms": 94.777, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-18e656d8741b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14900, + "latency_ms": 80.423, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-19f05b318577", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "found": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 57622, + "latency_ms": 67.904, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-1ca619fba9f2", + "path_leak": true, + "required_files": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 160932, + "latency_ms": 91.029, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-1d00d5a8b1af", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159689, + "latency_ms": 92.155, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-21839f193280", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [ + "app/layout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 34162, + "latency_ms": 27.701, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-2451fc913f88", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 131913, + "latency_ms": 84.353, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-2991f2b749b6", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 57622, + "latency_ms": 69.089, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-2d81c1441e17", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 191617, + "latency_ms": 97.374, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-2e7392f7dccd", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32357, + "latency_ms": 83.378, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-3224f6662580", + "path_leak": true, + "required_files": [ + "apps/mobile/src/components/preview-pane.d.ts", + "apps/mobile/src/components/preview-pane.ts" + ], + "found": [], + "missed": [ + "apps/mobile/src/components/preview-pane.d.ts", + "apps/mobile/src/components/preview-pane.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123413, + "latency_ms": 25.71, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-32e6fd1b5c87", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37794, + "latency_ms": 34.219, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-40426bd7a03a", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 144567, + "latency_ms": 108.439, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-41ad25bba286", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 179520, + "latency_ms": 143.263, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-464fe2d9d985", + "path_leak": false, + "required_files": [ + "biome.json" + ], + "found": [ + "biome.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 38099, + "latency_ms": 28.547, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-4b3332684075", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 143425, + "latency_ms": 84.998, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-4c00f66be11b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 147821, + "latency_ms": 94.242, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-4efb5e0e9f6c", + "path_leak": false, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38341, + "latency_ms": 27.369, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-4f8e172c77c0", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18532, + "latency_ms": 64.019, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-58ce619558e4", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 181851, + "latency_ms": 100.543, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5947cb9235a4", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32357, + "latency_ms": 59.529, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5add561d3a1c", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "found": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "missed": [], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.8, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 184098, + "latency_ms": 97.219, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5c274bfb8d8c", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68366, + "latency_ms": 30.781, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5c737aa349c4", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [ + "app/layout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 34196, + "latency_ms": 32.59, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5e9dd3d3a183", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 39535, + "latency_ms": 34.563, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5f057c9840e6", + "path_leak": true, + "required_files": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [ + "foundry/ops/config/projects.json" + ], + "missed": [ + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.1429, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 3, + "tokens_delivered": 32020, + "latency_ms": 65.732, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5ff8c1d23d1a", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 57549, + "latency_ms": 62.188, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-607ffead8d59", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 181851, + "latency_ms": 87.441, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-6410c21e39fb", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 180464, + "latency_ms": 83.618, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-6806c601cd49", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 91941, + "latency_ms": 30.084, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-6a61e9774eee", + "path_leak": true, + "required_files": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/design-workflow.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/design-workflow.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46309, + "latency_ms": 62.997, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-6e4082265380", + "path_leak": true, + "required_files": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43420, + "latency_ms": 69.285, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-70a328abccdb", + "path_leak": false, + "required_files": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "found": [], + "missed": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42408, + "latency_ms": 83.072, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-747d98b7face", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18532, + "latency_ms": 72.48, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-756936a2820c", + "path_leak": false, + "required_files": [ + "scripts/prepare-node-pty.mjs" + ], + "found": [], + "missed": [ + "scripts/prepare-node-pty.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 119636, + "latency_ms": 37.234, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-834566d04be7", + "path_leak": true, + "required_files": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "found": [], + "missed": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44808, + "latency_ms": 99.302, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-864bb66e08c9", + "path_leak": false, + "required_files": [ + "foundry/ops/config/projects.json" + ], + "found": [ + "foundry/ops/config/projects.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 179290, + "latency_ms": 114.515, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-8c06ab66e57f", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 147535, + "latency_ms": 85.401, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-90d2bee838d8", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "missed": [ + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 142904, + "latency_ms": 109.537, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-93e088e6145f", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43420, + "latency_ms": 61.15, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-98062c56ade9", + "path_leak": true, + "required_files": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/project-catalog.test.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "found": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "missed": [ + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 191617, + "latency_ms": 82.679, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-9afe0805eedb", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 179520, + "latency_ms": 85.827, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-9d823e16ea99", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/search-console.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/search-console.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 147424, + "latency_ms": 94.008, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-9e0367722215", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 142313, + "latency_ms": 90.935, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-9f748e8bfd7a", + "path_leak": false, + "required_files": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts", + "packages/protocol/src/index.ts" + ], + "found": [], + "missed": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts", + "packages/protocol/src/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123413, + "latency_ms": 37.643, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-a29095247805", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42408, + "latency_ms": 64.443, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-a67cc73f8c17", + "path_leak": true, + "required_files": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "found": [], + "missed": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 56617, + "latency_ms": 37.776, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-a766a9fe6db6", + "path_leak": true, + "required_files": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "found": [], + "missed": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34699, + "latency_ms": 44.074, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-a7e020f5380c", + "path_leak": true, + "required_files": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "found": [ + "fleet-ops/config/projects.json" + ], + "missed": [ + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.1429, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 5, + "tokens_delivered": 62450, + "latency_ms": 37.793, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-aa702f549db8", + "path_leak": true, + "required_files": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "found": [ + "apps/backend/config/projects.json" + ], + "missed": [ + "apps/backend/test/catalog-boundary.test.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71914, + "latency_ms": 165.033, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-abf1e1c3dcf9", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 138229, + "latency_ms": 78.879, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-ac2d8a08ccf7", + "path_leak": true, + "required_files": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67468, + "latency_ms": 43.524, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-ad8c488454cf", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/git-health.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/git-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68366, + "latency_ms": 31.176, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-b311494c1853", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "missed": [ + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 141888, + "latency_ms": 87.931, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-b43186e6c2d6", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 142313, + "latency_ms": 91.117, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-b460bbee477e", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/page.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/page.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159658, + "latency_ms": 108.849, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-b963e53c1599", + "path_leak": false, + "required_files": [ + "foundry/ops/config/marketing-program.json" + ], + "found": [ + "foundry/ops/config/marketing-program.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 147535, + "latency_ms": 79.311, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-b9a9b7fb722e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 191618, + "latency_ms": 95.333, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c055fb854a9c", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 141888, + "latency_ms": 95.718, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c05c3a7fd02e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145710, + "latency_ms": 83.633, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c30d08ebb49f", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67192, + "latency_ms": 89.89, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c3cb6dd9d680", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 143521, + "latency_ms": 87.275, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c4f7e2d22b0c", + "path_leak": true, + "required_files": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159658, + "latency_ms": 98.144, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c50c5280c323", + "path_leak": true, + "required_files": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 161810, + "latency_ms": 110.608, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c99064f83b83", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159680, + "latency_ms": 100.156, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-ca50c4d300dd", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "missed": [ + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 144090, + "latency_ms": 84.327, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-d906a7364744", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 184565, + "latency_ms": 90.269, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-e132f96c25ac", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159680, + "latency_ms": 96.518, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-e261ca6aa79e", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10973, + "latency_ms": 55.407, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-e288e502d891", + "path_leak": false, + "required_files": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "found": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 123181, + "latency_ms": 26.096, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-e4a52d0adfb6", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159680, + "latency_ms": 98.174, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-e7dec365bcf9", + "path_leak": false, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140123, + "latency_ms": 86.77, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-e8113343338f", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 184580, + "latency_ms": 109.992, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-eaaf7a67fc35", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "found": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json" + ], + "missed": [ + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 61019, + "latency_ms": 88.198, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-efdb446d024f", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [ + "foundry/ops/test/public-products.test.mjs" + ], + "missed": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 145934, + "latency_ms": 90.813, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-f157706d6ddc", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "missed": [ + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 146213, + "latency_ms": 85.31, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-f27818367be8", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159680, + "latency_ms": 114.169, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-f2b275130ace", + "path_leak": true, + "required_files": [ + "fleet-ops/config/marketing-program.json" + ], + "found": [ + "fleet-ops/config/marketing-program.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 67468, + "latency_ms": 46.652, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-f7557d39e6e6", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [ + "foundry/ops/test/public-products.test.mjs" + ], + "missed": [ + "foundry/apps/public/public-directory/public/_redirects" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 146153, + "latency_ms": 95.51, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-f8b07be6f191", + "path_leak": true, + "required_files": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "found": [ + "apps/mobile/app.json" + ], + "missed": [ + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 38020, + "latency_ms": 23.309, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-fa0e527907ad", + "path_leak": true, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67192, + "latency_ms": 90.834, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-00d7bb046471", + "path_leak": false, + "required_files": [ + "apps/backend/config/projects.json" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3338, + "latency_ms": 798.145, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-010e4aebe75e", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3751, + "latency_ms": 2606.57, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-054409454be6", + "path_leak": false, + "required_files": [ + ".gitmodules" + ], + "found": [], + "missed": [ + ".gitmodules" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3495, + "latency_ms": 1436.736, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-071417754558", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/next.config.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3436, + "latency_ms": 5736.673, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-07ad25caddd4", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3976, + "latency_ms": 3741.735, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-0ba9b5f5948b", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3508, + "latency_ms": 5996.747, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-0baeb9010cb3", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 3743, + "latency_ms": 4615.954, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-132847b20d6d", + "path_leak": false, + "required_files": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3455, + "latency_ms": 5547.86, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-18e656d8741b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4248, + "latency_ms": 2629.424, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-19f05b318577", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "found": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3557, + "latency_ms": 3712.407, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-1ca619fba9f2", + "path_leak": true, + "required_files": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "found": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3588, + "latency_ms": 5270.725, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-1d00d5a8b1af", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4086, + "latency_ms": 5549.837, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-21839f193280", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [], + "missed": [ + "app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3360, + "latency_ms": 644.496, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-2451fc913f88", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 3842, + "latency_ms": 4414.835, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-2991f2b749b6", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4048, + "latency_ms": 3330.043, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-2d81c1441e17", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "found": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3736, + "latency_ms": 5962.163, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-2e7392f7dccd", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4076, + "latency_ms": 3070.559, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-3224f6662580", + "path_leak": true, + "required_files": [ + "apps/mobile/src/components/preview-pane.d.ts", + "apps/mobile/src/components/preview-pane.ts" + ], + "found": [ + "apps/mobile/src/components/preview-pane.ts" + ], + "missed": [ + "apps/mobile/src/components/preview-pane.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3190, + "latency_ms": 712.479, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-32e6fd1b5c87", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4209, + "latency_ms": 950.305, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-40426bd7a03a", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3659, + "latency_ms": 5271.837, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-41ad25bba286", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4320, + "latency_ms": 6012.693, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-464fe2d9d985", + "path_leak": false, + "required_files": [ + "biome.json" + ], + "found": [], + "missed": [ + "biome.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3947, + "latency_ms": 655.49, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-4b3332684075", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3953, + "latency_ms": 5186.875, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-4c00f66be11b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3788, + "latency_ms": 5291.531, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-4efb5e0e9f6c", + "path_leak": false, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3465, + "latency_ms": 949.815, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-4f8e172c77c0", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3737, + "latency_ms": 2737.85, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-58ce619558e4", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs" + ], + "missed": [ + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3626, + "latency_ms": 6375.408, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-5947cb9235a4", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4187, + "latency_ms": 3090.115, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-5add561d3a1c", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3357, + "latency_ms": 2886.106, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-5c274bfb8d8c", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4186, + "latency_ms": 1472.393, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-5c737aa349c4", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [], + "missed": [ + "app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3608, + "latency_ms": 632.762, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-5e9dd3d3a183", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4044, + "latency_ms": 1116.684, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-5f057c9840e6", + "path_leak": true, + "required_files": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4100, + "latency_ms": 3097.882, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-5ff8c1d23d1a", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3381, + "latency_ms": 4355.834, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-607ffead8d59", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs" + ], + "missed": [ + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3715, + "latency_ms": 6254.868, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-6410c21e39fb", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3500, + "latency_ms": 4044.15, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-6806c601cd49", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3090, + "latency_ms": 2801.201, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-6a61e9774eee", + "path_leak": true, + "required_files": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/design-workflow.mjs" + ], + "found": [ + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/design-workflow.mjs" + ], + "missed": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/scripts/agent-stack.sh" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4588, + "latency_ms": 4200.53, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-6e4082265380", + "path_leak": true, + "required_files": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3895, + "latency_ms": 4686.413, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-70a328abccdb", + "path_leak": false, + "required_files": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "found": [ + "foundry/apps/setline/worker/schema.ts" + ], + "missed": [ + "foundry/apps/setline/tests/rendered-html.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 3972, + "latency_ms": 4981.256, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-747d98b7face", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3841, + "latency_ms": 3162.169, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-756936a2820c", + "path_leak": false, + "required_files": [ + "scripts/prepare-node-pty.mjs" + ], + "found": [ + "scripts/prepare-node-pty.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 14, + "tokens_delivered": 3511, + "latency_ms": 960.59, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-834566d04be7", + "path_leak": true, + "required_files": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "found": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js" + ], + "missed": [ + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3355, + "latency_ms": 4296.329, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-864bb66e08c9", + "path_leak": false, + "required_files": [ + "foundry/ops/config/projects.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3890, + "latency_ms": 7660.804, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-8c06ab66e57f", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3696, + "latency_ms": 6495.151, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-90d2bee838d8", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3798, + "latency_ms": 4035.595, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-93e088e6145f", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "found": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3192, + "latency_ms": 3558.013, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-98062c56ade9", + "path_leak": true, + "required_files": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/project-catalog.test.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "found": [ + "foundry/ops/test/project-catalog.test.mjs" + ], + "missed": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.2, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 17, + "tokens_delivered": 3544, + "latency_ms": 5190.684, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-9afe0805eedb", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "found": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs" + ], + "missed": [ + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 3634, + "latency_ms": 5321.193, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-9d823e16ea99", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/search-console.mjs" + ], + "found": [ + "foundry/ops/lib/search-console.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3928, + "latency_ms": 4427.806, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-9e0367722215", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/lib/founder-control/connections.mjs" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3842, + "latency_ms": 4629.5, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-9f748e8bfd7a", + "path_leak": false, + "required_files": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts", + "packages/protocol/src/index.ts" + ], + "found": [ + "packages/protocol/src/index.ts" + ], + "missed": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 3584, + "latency_ms": 662.928, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-a29095247805", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2895, + "latency_ms": 3588.827, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-a67cc73f8c17", + "path_leak": true, + "required_files": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "found": [ + "fleet-ops/lib/public-products.mjs" + ], + "missed": [ + "fleet-ops/config/public-products.json", + "fleet-ops/public/products.json" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 4100, + "latency_ms": 2425.728, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-a766a9fe6db6", + "path_leak": true, + "required_files": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "found": [ + "lib/api-timing.ts" + ], + "missed": [ + "app/data/page.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 3464, + "latency_ms": 587.233, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-a7e020f5380c", + "path_leak": true, + "required_files": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "found": [ + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "missed": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.2857, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.1429, + "recall_at_4000_tokens": 0.1429, + "recall_at_16000_tokens": 0.2857, + "first_hit_rank": 1, + "tokens_delivered": 4008, + "latency_ms": 2681.036, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-aa702f549db8", + "path_leak": true, + "required_files": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3664, + "latency_ms": 762.382, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-abf1e1c3dcf9", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/lib/founder-control/connections.mjs" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 3, + "tokens_delivered": 3333, + "latency_ms": 4675.639, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-ac2d8a08ccf7", + "path_leak": true, + "required_files": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "found": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs" + ], + "missed": [ + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3832, + "latency_ms": 1537.453, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-ad8c488454cf", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/git-health.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/git-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4122, + "latency_ms": 1561.428, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-b311494c1853", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "missed": [ + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.1579, + "recall_at_1000_tokens": 0.4, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 4, + "tokens_delivered": 3395, + "latency_ms": 5933.085, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-b43186e6c2d6", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3605, + "latency_ms": 6181.53, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-b460bbee477e", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/page.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/page.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3182, + "latency_ms": 8372.816, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-b963e53c1599", + "path_leak": false, + "required_files": [ + "foundry/ops/config/marketing-program.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/marketing-program.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3837, + "latency_ms": 5969.737, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-b9a9b7fb722e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4203, + "latency_ms": 7228.681, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-c055fb854a9c", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "found": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 3933, + "latency_ms": 6037.395, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-c05c3a7fd02e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "found": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "missed": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 3918, + "latency_ms": 6877.868, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-c30d08ebb49f", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/lib/founder-control/connections.mjs" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4257, + "latency_ms": 5334.548, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-c3cb6dd9d680", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs" + ], + "missed": [ + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 4039, + "latency_ms": 6428.163, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-c4f7e2d22b0c", + "path_leak": true, + "required_files": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "found": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py" + ], + "missed": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "recall_at_5": 0.375, + "precision_at_5": 0.6, + "recall_at_10": 0.375, + "precision_at_10": 0.3, + "recall_at_20": 0.375, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.375, + "recall_at_4000_tokens": 0.375, + "recall_at_16000_tokens": 0.375, + "first_hit_rank": 1, + "tokens_delivered": 3630, + "latency_ms": 7094.389, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-c50c5280c323", + "path_leak": true, + "required_files": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4169, + "latency_ms": 6714.334, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-c99064f83b83", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 4339, + "latency_ms": 7498.794, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-ca50c4d300dd", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs" + ], + "missed": [ + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 5, + "tokens_delivered": 3621, + "latency_ms": 5467.929, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-d906a7364744", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3723, + "latency_ms": 6889.164, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-e132f96c25ac", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3241, + "latency_ms": 6355.568, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-e261ca6aa79e", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "found": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3937, + "latency_ms": 2909.153, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-e288e502d891", + "path_leak": false, + "required_files": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "found": [], + "missed": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3697, + "latency_ms": 768.457, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-e4a52d0adfb6", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3939, + "latency_ms": 6066.866, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-e7dec365bcf9", + "path_leak": false, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3846, + "latency_ms": 5658.513, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-e8113343338f", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "found": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3890, + "latency_ms": 3138.549, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-eaaf7a67fc35", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3669, + "latency_ms": 4720.14, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-efdb446d024f", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [ + "foundry/apps/public/public-directory/src/data/publicRoutes.ts" + ], + "missed": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 3681, + "latency_ms": 5720.68, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-f157706d6ddc", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/scripts/search-console-collect.mjs" + ], + "missed": [ + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 7, + "tokens_delivered": 4197, + "latency_ms": 5725.887, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-f27818367be8", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3733, + "latency_ms": 6724.929, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-f2b275130ace", + "path_leak": true, + "required_files": [ + "fleet-ops/config/marketing-program.json" + ], + "found": [], + "missed": [ + "fleet-ops/config/marketing-program.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3413, + "latency_ms": 2842.706, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-f7557d39e6e6", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2643, + "latency_ms": 6442.375, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-f8b07be6f191", + "path_leak": true, + "required_files": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "found": [], + "missed": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3963, + "latency_ms": 766.007, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "site-health-fa0e527907ad", + "path_leak": true, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 4110, + "latency_ms": 4816.603, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-00d7bb046471", + "path_leak": false, + "required_files": [ + "apps/backend/config/projects.json" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 552, + "latency_ms": 636.676, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-010e4aebe75e", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 549, + "latency_ms": 1093.488, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-054409454be6", + "path_leak": false, + "required_files": [ + ".gitmodules" + ], + "found": [], + "missed": [ + ".gitmodules" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 494, + "latency_ms": 555.375, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-071417754558", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/next.config.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 488, + "latency_ms": 2838.736, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-07ad25caddd4", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 434, + "latency_ms": 2464.43, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-0ba9b5f5948b", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 553, + "latency_ms": 2979.78, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-0baeb9010cb3", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 587, + "latency_ms": 6464.317, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-132847b20d6d", + "path_leak": false, + "required_files": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 602, + "latency_ms": 5138.305, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-18e656d8741b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 554, + "latency_ms": 3540.768, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-19f05b318577", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "found": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 562, + "latency_ms": 2296.94, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-1ca619fba9f2", + "path_leak": true, + "required_files": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "found": [ + "foundry/helpers/chatgpt-connections/src/activation.ts" + ], + "missed": [ + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 583, + "latency_ms": 3608.674, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-1d00d5a8b1af", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 468, + "latency_ms": 5101.505, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-21839f193280", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [], + "missed": [ + "app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 458, + "latency_ms": 797.4, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-2451fc913f88", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 616, + "latency_ms": 3845.723, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-2991f2b749b6", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 434, + "latency_ms": 3235.233, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-2d81c1441e17", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "found": [ + "foundry/ops/lib/posthog-outcomes.mjs" + ], + "missed": [ + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 670, + "latency_ms": 4218.448, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-2e7392f7dccd", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 658, + "latency_ms": 3260.784, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-3224f6662580", + "path_leak": true, + "required_files": [ + "apps/mobile/src/components/preview-pane.d.ts", + "apps/mobile/src/components/preview-pane.ts" + ], + "found": [], + "missed": [ + "apps/mobile/src/components/preview-pane.d.ts", + "apps/mobile/src/components/preview-pane.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 544, + "latency_ms": 1021.451, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-32e6fd1b5c87", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 417, + "latency_ms": 1030.704, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-40426bd7a03a", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 521, + "latency_ms": 3479.669, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-41ad25bba286", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 498, + "latency_ms": 3187.942, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-464fe2d9d985", + "path_leak": false, + "required_files": [ + "biome.json" + ], + "found": [], + "missed": [ + "biome.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 322, + "latency_ms": 1165.15, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-4b3332684075", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 711, + "latency_ms": 3105.104, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-4c00f66be11b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 734, + "latency_ms": 3775.217, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-4efb5e0e9f6c", + "path_leak": false, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 432, + "latency_ms": 883.178, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-4f8e172c77c0", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 533, + "latency_ms": 3077.456, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-58ce619558e4", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 629, + "latency_ms": 5417.407, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-5947cb9235a4", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 424, + "latency_ms": 3227.912, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-5add561d3a1c", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 702, + "latency_ms": 1728.132, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-5c274bfb8d8c", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 480, + "latency_ms": 1446.06, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-5c737aa349c4", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [], + "missed": [ + "app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 433, + "latency_ms": 1639.082, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-5e9dd3d3a183", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 491, + "latency_ms": 844.781, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-5f057c9840e6", + "path_leak": true, + "required_files": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 691, + "latency_ms": 2280.135, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-5ff8c1d23d1a", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 447, + "latency_ms": 2890.394, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-607ffead8d59", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 629, + "latency_ms": 4399.311, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-6410c21e39fb", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 695, + "latency_ms": 2467.617, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-6806c601cd49", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 581, + "latency_ms": 1172.333, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-6a61e9774eee", + "path_leak": true, + "required_files": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/design-workflow.mjs" + ], + "found": [ + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh" + ], + "missed": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/scripts/design-workflow.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 532, + "latency_ms": 2736.819, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-6e4082265380", + "path_leak": true, + "required_files": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 571, + "latency_ms": 3595.282, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-70a328abccdb", + "path_leak": false, + "required_files": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "found": [], + "missed": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 598, + "latency_ms": 3588.475, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-747d98b7face", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 458, + "latency_ms": 2499.207, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-756936a2820c", + "path_leak": false, + "required_files": [ + "scripts/prepare-node-pty.mjs" + ], + "found": [], + "missed": [ + "scripts/prepare-node-pty.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 514, + "latency_ms": 1483.962, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-834566d04be7", + "path_leak": true, + "required_files": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "found": [], + "missed": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 616, + "latency_ms": 2577.035, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-864bb66e08c9", + "path_leak": false, + "required_files": [ + "foundry/ops/config/projects.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 579, + "latency_ms": 3599.979, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-8c06ab66e57f", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 634, + "latency_ms": 2965.045, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-90d2bee838d8", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 697, + "latency_ms": 2865.556, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-93e088e6145f", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "found": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 449, + "latency_ms": 1583.948, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-98062c56ade9", + "path_leak": true, + "required_files": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/project-catalog.test.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "found": [ + "foundry/ops/test/project-catalog.test.mjs" + ], + "missed": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "recall_at_5": 0.2, + "precision_at_5": 1, + "recall_at_10": 0.2, + "precision_at_10": 1, + "recall_at_20": 0.2, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 617, + "latency_ms": 2224.146, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-9afe0805eedb", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 605, + "latency_ms": 2214.374, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-9d823e16ea99", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/search-console.mjs" + ], + "found": [ + "foundry/ops/lib/search-console.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 558, + "latency_ms": 2224.33, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-9e0367722215", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0.25, + "precision_at_5": 1, + "recall_at_10": 0.25, + "precision_at_10": 1, + "recall_at_20": 0.25, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 625, + "latency_ms": 2393.824, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-9f748e8bfd7a", + "path_leak": false, + "required_files": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts", + "packages/protocol/src/index.ts" + ], + "found": [], + "missed": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts", + "packages/protocol/src/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 649, + "latency_ms": 592.149, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-a29095247805", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 498, + "latency_ms": 3025.464, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-a67cc73f8c17", + "path_leak": true, + "required_files": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "found": [], + "missed": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 586, + "latency_ms": 2025.498, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-a766a9fe6db6", + "path_leak": true, + "required_files": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "found": [], + "missed": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 421, + "latency_ms": 535.91, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-a7e020f5380c", + "path_leak": true, + "required_files": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 462, + "latency_ms": 2089.927, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-aa702f549db8", + "path_leak": true, + "required_files": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 544, + "latency_ms": 566.416, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-abf1e1c3dcf9", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.5, + "recall_at_10": 0.2, + "precision_at_10": 0.5, + "recall_at_20": 0.2, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 459, + "latency_ms": 4586.471, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-ac2d8a08ccf7", + "path_leak": true, + "required_files": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 521, + "latency_ms": 1044.27, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-ad8c488454cf", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/git-health.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/git-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 543, + "latency_ms": 995.084, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-b311494c1853", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 504, + "latency_ms": 3148.136, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-b43186e6c2d6", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [ + "foundry/helpers/drank/app/globals.css" + ], + "missed": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 510, + "latency_ms": 2733.561, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-b460bbee477e", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/page.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/page.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 512, + "latency_ms": 2677.427, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-b963e53c1599", + "path_leak": false, + "required_files": [ + "foundry/ops/config/marketing-program.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/marketing-program.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 530, + "latency_ms": 2608.291, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-b9a9b7fb722e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 641, + "latency_ms": 3162.664, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-c055fb854a9c", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 806, + "latency_ms": 3353.7, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-c05c3a7fd02e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 653, + "latency_ms": 3018.794, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-c30d08ebb49f", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.5, + "recall_at_10": 0.1667, + "precision_at_10": 0.5, + "recall_at_20": 0.1667, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 2, + "tokens_delivered": 622, + "latency_ms": 2470.209, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-c3cb6dd9d680", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 552, + "latency_ms": 3296.485, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-c4f7e2d22b0c", + "path_leak": true, + "required_files": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 482, + "latency_ms": 2268.014, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-c50c5280c323", + "path_leak": true, + "required_files": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 609, + "latency_ms": 2978.614, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-c99064f83b83", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 598, + "latency_ms": 3765.806, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-ca50c4d300dd", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 560, + "latency_ms": 3880.846, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-d906a7364744", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 471, + "latency_ms": 5614.048, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-e132f96c25ac", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 588, + "latency_ms": 4808.697, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-e261ca6aa79e", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 627, + "latency_ms": 2014.185, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-e288e502d891", + "path_leak": false, + "required_files": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "found": [], + "missed": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3116, + "latency_ms": 775.271, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-e4a52d0adfb6", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [ + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "missed": [ + "foundry/ops/scripts/deploy-health.sh" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 503, + "latency_ms": 3150.222, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-e7dec365bcf9", + "path_leak": false, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 509, + "latency_ms": 2483.447, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-e8113343338f", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "found": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 659, + "latency_ms": 1002.477, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-eaaf7a67fc35", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 473, + "latency_ms": 2248.437, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-efdb446d024f", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [ + "foundry/ops/test/public-products.test.mjs" + ], + "missed": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 741, + "latency_ms": 2172.288, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-f157706d6ddc", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "found": [ + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.5, + "recall_at_10": 0.2, + "precision_at_10": 0.5, + "recall_at_20": 0.2, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 687, + "latency_ms": 2448.605, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-f27818367be8", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 553, + "latency_ms": 2334.479, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-f2b275130ace", + "path_leak": true, + "required_files": [ + "fleet-ops/config/marketing-program.json" + ], + "found": [], + "missed": [ + "fleet-ops/config/marketing-program.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 477, + "latency_ms": 717.701, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-f7557d39e6e6", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 770, + "latency_ms": 2499.404, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-f8b07be6f191", + "path_leak": true, + "required_files": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "found": [], + "missed": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 598, + "latency_ms": 733.392, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "site-health-fa0e527907ad", + "path_leak": true, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 599, + "latency_ms": 2652.988, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-00d7bb046471", + "path_leak": false, + "required_files": [ + "apps/backend/config/projects.json" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1347.407, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-010e4aebe75e", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1012.648, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-054409454be6", + "path_leak": false, + "required_files": [ + ".gitmodules" + ], + "found": [], + "missed": [ + ".gitmodules" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1052.695, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-071417754558", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/next.config.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 891.303, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-07ad25caddd4", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3264, + "latency_ms": 711.621, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-0ba9b5f5948b", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 709.067, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-0baeb9010cb3", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 638.101, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-132847b20d6d", + "path_leak": false, + "required_files": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 649.497, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-18e656d8741b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 718.026, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-19f05b318577", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 794.956, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-1ca619fba9f2", + "path_leak": true, + "required_files": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 932.237, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-1d00d5a8b1af", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 871.817, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-21839f193280", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [], + "missed": [ + "app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 901.072, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-2451fc913f88", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 916.55, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-2991f2b749b6", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 343, + "latency_ms": 581.176, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-2d81c1441e17", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 903.586, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-2e7392f7dccd", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 830.108, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-3224f6662580", + "path_leak": true, + "required_files": [ + "apps/mobile/src/components/preview-pane.d.ts", + "apps/mobile/src/components/preview-pane.ts" + ], + "found": [], + "missed": [ + "apps/mobile/src/components/preview-pane.d.ts", + "apps/mobile/src/components/preview-pane.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 600.933, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-32e6fd1b5c87", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 844.824, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-40426bd7a03a", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 775.506, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-41ad25bba286", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 708.779, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-464fe2d9d985", + "path_leak": false, + "required_files": [ + "biome.json" + ], + "found": [], + "missed": [ + "biome.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1003.473, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-4b3332684075", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 683.68, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-4c00f66be11b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 707.65, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-4efb5e0e9f6c", + "path_leak": false, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 608.676, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-4f8e172c77c0", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 703.437, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-58ce619558e4", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 679.715, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-5947cb9235a4", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 588.005, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-5add561d3a1c", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 567.408, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-5c274bfb8d8c", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 511.368, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-5c737aa349c4", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [], + "missed": [ + "app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 517.374, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-5e9dd3d3a183", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 549.186, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-5f057c9840e6", + "path_leak": true, + "required_files": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 546.159, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-5ff8c1d23d1a", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 569.964, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-607ffead8d59", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 343, + "latency_ms": 602.773, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-6410c21e39fb", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 648.674, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-6806c601cd49", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 600.718, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-6a61e9774eee", + "path_leak": true, + "required_files": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/design-workflow.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/design-workflow.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 511.885, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-6e4082265380", + "path_leak": true, + "required_files": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 503.264, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-70a328abccdb", + "path_leak": false, + "required_files": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "found": [], + "missed": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 554.817, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-747d98b7face", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 531.301, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-756936a2820c", + "path_leak": false, + "required_files": [ + "scripts/prepare-node-pty.mjs" + ], + "found": [], + "missed": [ + "scripts/prepare-node-pty.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 397.876, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-834566d04be7", + "path_leak": true, + "required_files": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "found": [], + "missed": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 557.03, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-864bb66e08c9", + "path_leak": false, + "required_files": [ + "foundry/ops/config/projects.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 568.334, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-8c06ab66e57f", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 508.652, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-90d2bee838d8", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 501.81, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-93e088e6145f", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 568.472, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-98062c56ade9", + "path_leak": true, + "required_files": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/project-catalog.test.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/project-catalog.test.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 511.041, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-9afe0805eedb", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 515.829, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-9d823e16ea99", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/search-console.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/search-console.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 400.586, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-9e0367722215", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 535.093, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-9f748e8bfd7a", + "path_leak": false, + "required_files": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts", + "packages/protocol/src/index.ts" + ], + "found": [], + "missed": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts", + "packages/protocol/src/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 504.231, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-a29095247805", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 406.261, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-a67cc73f8c17", + "path_leak": true, + "required_files": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "found": [], + "missed": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 523.25, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-a766a9fe6db6", + "path_leak": true, + "required_files": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "found": [], + "missed": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 508.527, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-a7e020f5380c", + "path_leak": true, + "required_files": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 551.715, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-aa702f549db8", + "path_leak": true, + "required_files": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 401.662, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-abf1e1c3dcf9", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 526.681, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-ac2d8a08ccf7", + "path_leak": true, + "required_files": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 502.267, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-ad8c488454cf", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/git-health.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/git-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 519.257, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-b311494c1853", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 506, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-b43186e6c2d6", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 506.465, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-b460bbee477e", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/page.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/page.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 486.851, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-b963e53c1599", + "path_leak": false, + "required_files": [ + "foundry/ops/config/marketing-program.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/marketing-program.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 490.944, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-b9a9b7fb722e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 392.564, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-c055fb854a9c", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 480.595, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-c05c3a7fd02e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 492.751, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-c30d08ebb49f", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 486.718, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-c3cb6dd9d680", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 396.935, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-c4f7e2d22b0c", + "path_leak": true, + "required_files": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 498.074, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-c50c5280c323", + "path_leak": true, + "required_files": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 503.167, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-c99064f83b83", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 386.46, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-ca50c4d300dd", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 520.444, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-d906a7364744", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 383.615, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-e132f96c25ac", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 480.357, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-e261ca6aa79e", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 430.027, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-e288e502d891", + "path_leak": false, + "required_files": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "found": [], + "missed": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 385.615, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-e4a52d0adfb6", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 397.649, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-e7dec365bcf9", + "path_leak": false, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 570.072, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-e8113343338f", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 380.138, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-eaaf7a67fc35", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 561.347, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-efdb446d024f", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 459.072, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-f157706d6ddc", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 488.667, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-f27818367be8", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 381.387, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-f2b275130ace", + "path_leak": true, + "required_files": [ + "fleet-ops/config/marketing-program.json" + ], + "found": [], + "missed": [ + "fleet-ops/config/marketing-program.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 489.644, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-f7557d39e6e6", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 431.641, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-f8b07be6f191", + "path_leak": true, + "required_files": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "found": [], + "missed": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 374.498, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "site-health-fa0e527907ad", + "path_leak": true, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 381.177, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/P-today-little-log.json b/benchmarks/context-retrieval/results/P-today-little-log.json new file mode 100644 index 00000000..4976f2a7 --- /dev/null +++ b/benchmarks/context-retrieval/results/P-today-little-log.json @@ -0,0 +1,6158 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "today-little-log", + "head": "93b48d03329b74df043f0b45468f96ac21197c41" + }, + "tier": "small", + "tier_code_files": 117, + "corpus_counts": { + "cases": 41, + "path_leak": 21, + "baseline_blind": 20, + "multi_file": 15 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.2616, + "control": 0.0035, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "CSS import order and PostCSS conflict", + "revision": "0458fb2e26f9d78c20c824b75b62f3c8cc5a9592", + "returned": [ + "src/pwa.ts", + "src/components/ui/hover-card.tsx", + "src/components/ui/toast.tsx", + "src/components/ui/sheet.tsx", + "src/components/ui/button.tsx" + ] + }, + { + "provider_id": "random-files", + "query": "pre-push runs tsc + build, not just lint", + "revision": "9631951e22885658c3fa72897305aff4bd3690a8", + "returned": [ + "src/components/ui/label.tsx", + "src/components/StreakBadge.tsx", + "renovate.json", + "src/lib/auth-client.ts", + "drizzle/migrations/0002_urgency_wave_b.sql" + ] + }, + { + "provider_id": "random-files", + "query": "consolidate sidebar, tolerant /api/profiles, craft-hours widget", + "revision": "4a09700d0f7f1b5afad3dd6136090f448403bddb", + "returned": [ + "src/vite-env.d.ts", + "src/pages/Goals.tsx", + "src/components/LifeScoreBadge.tsx", + "api/_lib/auth-instance.ts", + "src/pages/Index.tsx" + ] + }, + { + "provider_id": "churn-ranked", + "query": "vendor @saas-maker/ai so Vercel builds", + "revision": "b297f57a34899bbca5463d4df60bc7acf40ba3c4", + "returned": [ + "src/pages/Index.tsx", + "package.json", + "src/App.tsx", + "src/components/AppSidebar.tsx", + "src/components/ScheduleMaker.tsx" + ] + }, + { + "provider_id": "churn-ranked", + "query": "route /api/auth/* subpaths to api/auth.ts", + "revision": "d3f3c964ec945f8b01a6c55acfe2fd19755a837a", + "returned": [ + "src/pages/Index.tsx", + "package.json", + "src/App.tsx", + "src/components/AppSidebar.tsx", + "vercel.json" + ] + }, + { + "provider_id": "semble", + "query": "Fix route smoke tests after score page consolidation", + "revision": "0ce78189844f08a63010d3a98a94efc1f06fa3ed", + "returned": [ + "src/config/monthlyScoreboards.ts", + "src/hooks/useScoreboard.ts", + "functions/api/scoreboard-items.ts", + "src/lib/scoreboardDefaults.ts", + "functions/api/scoreboard-logs.ts" + ] + }, + { + "provider_id": "semble", + "query": "chat scroll, better system prompt, persist messages", + "revision": "5bd849d4d97cc76bfd58c4b5770fb63b762f6fe1", + "returned": [ + "api/chat.ts", + "src/hooks/useChat.ts", + "api/_lib/auth.ts", + "src/hooks/useSchedule.ts", + "src/db/schema.ts" + ] + }, + { + "provider_id": "semble", + "query": "restore original URL path for Better Auth callback routing", + "revision": "0af1c2adc060849089425b4bcb7b2118b3fd9ad3", + "returned": [ + "api/_lib/auth.ts", + "src/lib/auth-client.ts", + "api/auth.ts", + "src/hooks/useAuth.ts", + "src/hooks/useChat.ts" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "vendor @saas-maker/ai so Vercel builds", + "revision": "b297f57a34899bbca5463d4df60bc7acf40ba3c4", + "returned": [ + "pnpm-lock.yaml", + "package-lock.json", + "package.json" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "use single auth.ts with rewrite for Better Auth catch-all", + "revision": "62d9d24a111619497a4398d2cba81a00e674bfe6", + "returned": [ + "vercel.json", + "tsconfig.json", + "docs/plans/2026-02-27-better-auth-migration-plan.md", + "package-lock.json", + "docs/plans/2026-02-27-better-auth-migration-design.md" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 41, + "mean_recall_at_5": 0.0035, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0157, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.097, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0035, + "mean_recall_at_16000_tokens": 0.0401, + "mean_precision_at_10": 0.0073, + "zero_hit_rate": 0.8293, + "median_tokens_delivered": 17981, + "median_latency_ms": 14.503, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 41, + "mean_recall_at_5": 0.0035, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0157, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.097, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0035, + "mean_recall_at_16000_tokens": 0.0401, + "mean_precision_at_10": 0.0073, + "zero_hit_rate": 0.8293, + "median_tokens_delivered": 17981, + "median_latency_ms": 14.503, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.0068, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0187, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1298, + "full_recall_at_20": 0.0952, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0068, + "mean_recall_at_16000_tokens": 0.0663, + "mean_precision_at_10": 0.0095, + "zero_hit_rate": 0.7619, + "median_tokens_delivered": 16346, + "median_latency_ms": 15.224, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 20, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0125, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0625, + "full_recall_at_20": 0.05, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0125, + "mean_precision_at_10": 0.005, + "zero_hit_rate": 0.9, + "median_tokens_delivered": 19008.5, + "median_latency_ms": 14.341, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 41 + }, + "resources": { + "baseline_rss_mb": 16819.5, + "peak_rss_mb": 17148.6, + "delta_rss_mb": 329.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 41, + "mean_recall_at_5": 0.0801, + "full_recall_at_5": 0.0244, + "mean_recall_at_10": 0.2041, + "full_recall_at_10": 0.122, + "mean_recall_at_20": 0.3017, + "full_recall_at_20": 0.2195, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0157, + "mean_precision_at_10": 0.0341, + "zero_hit_rate": 0.561, + "median_tokens_delivered": 140450, + "median_latency_ms": 23.724, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 41, + "mean_recall_at_5": 0.0801, + "full_recall_at_5": 0.0244, + "mean_recall_at_10": 0.2041, + "full_recall_at_10": 0.122, + "mean_recall_at_20": 0.3017, + "full_recall_at_20": 0.2195, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0157, + "mean_precision_at_10": 0.0341, + "zero_hit_rate": 0.561, + "median_tokens_delivered": 140450, + "median_latency_ms": 23.724, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.0612, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.208, + "full_recall_at_10": 0.0952, + "mean_recall_at_20": 0.339, + "full_recall_at_20": 0.2381, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0068, + "mean_precision_at_10": 0.0429, + "zero_hit_rate": 0.4762, + "median_tokens_delivered": 217508, + "median_latency_ms": 23.265, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 20, + "mean_recall_at_5": 0.1, + "full_recall_at_5": 0.05, + "mean_recall_at_10": 0.2, + "full_recall_at_10": 0.15, + "mean_recall_at_20": 0.2625, + "full_recall_at_20": 0.2, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.025, + "mean_precision_at_10": 0.025, + "zero_hit_rate": 0.65, + "median_tokens_delivered": 139356.5, + "median_latency_ms": 23.8555, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 41 + }, + "resources": { + "baseline_rss_mb": 16320.2, + "peak_rss_mb": 16814.2, + "delta_rss_mb": 494.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 41, + "mean_recall_at_5": 0.2352, + "full_recall_at_5": 0.1951, + "mean_recall_at_10": 0.2616, + "full_recall_at_10": 0.1951, + "mean_recall_at_20": 0.2616, + "full_recall_at_20": 0.1951, + "mean_recall_at_1000_tokens": 0.1986, + "mean_recall_at_4000_tokens": 0.2616, + "mean_recall_at_16000_tokens": 0.2616, + "mean_precision_at_10": 0.0444, + "zero_hit_rate": 0.6341, + "median_tokens_delivered": 3200, + "median_latency_ms": 802.772, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 41, + "mean_recall_at_5": 0.2352, + "full_recall_at_5": 0.1951, + "mean_recall_at_10": 0.2616, + "full_recall_at_10": 0.1951, + "mean_recall_at_20": 0.2616, + "full_recall_at_20": 0.1951, + "mean_recall_at_1000_tokens": 0.1986, + "mean_recall_at_4000_tokens": 0.2616, + "mean_recall_at_16000_tokens": 0.2616, + "mean_precision_at_10": 0.0444, + "zero_hit_rate": 0.6341, + "median_tokens_delivered": 3200, + "median_latency_ms": 802.772, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.2687, + "full_recall_at_5": 0.1905, + "mean_recall_at_10": 0.2846, + "full_recall_at_10": 0.1905, + "mean_recall_at_20": 0.2846, + "full_recall_at_20": 0.1905, + "mean_recall_at_1000_tokens": 0.2449, + "mean_recall_at_4000_tokens": 0.2846, + "mean_recall_at_16000_tokens": 0.2846, + "mean_precision_at_10": 0.0528, + "zero_hit_rate": 0.5714, + "median_tokens_delivered": 3134, + "median_latency_ms": 826.242, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 20, + "mean_recall_at_5": 0.2, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.2375, + "full_recall_at_10": 0.2, + "mean_recall_at_20": 0.2375, + "full_recall_at_20": 0.2, + "mean_recall_at_1000_tokens": 0.15, + "mean_recall_at_4000_tokens": 0.2375, + "mean_recall_at_16000_tokens": 0.2375, + "mean_precision_at_10": 0.0355, + "zero_hit_rate": 0.7, + "median_tokens_delivered": 3319.5, + "median_latency_ms": 798.2485, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 41 + }, + "resources": { + "baseline_rss_mb": 16505.9, + "peak_rss_mb": 18428.9, + "delta_rss_mb": 1922.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "jcodemunch", + "summary": { + "all": { + "cases": 41, + "mean_recall_at_5": 0.0488, + "full_recall_at_5": 0.0488, + "mean_recall_at_10": 0.0488, + "full_recall_at_10": 0.0488, + "mean_recall_at_20": 0.0488, + "full_recall_at_20": 0.0488, + "mean_recall_at_1000_tokens": 0.0488, + "mean_recall_at_4000_tokens": 0.0488, + "mean_recall_at_16000_tokens": 0.0488, + "mean_precision_at_10": 0.0163, + "zero_hit_rate": 0.9512, + "median_tokens_delivered": 636, + "median_latency_ms": 869.155, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 6 + }, + "baseline_missed": { + "cases": 41, + "mean_recall_at_5": 0.0488, + "full_recall_at_5": 0.0488, + "mean_recall_at_10": 0.0488, + "full_recall_at_10": 0.0488, + "mean_recall_at_20": 0.0488, + "full_recall_at_20": 0.0488, + "mean_recall_at_1000_tokens": 0.0488, + "mean_recall_at_4000_tokens": 0.0488, + "mean_recall_at_16000_tokens": 0.0488, + "mean_precision_at_10": 0.0163, + "zero_hit_rate": 0.9512, + "median_tokens_delivered": 636, + "median_latency_ms": 869.155, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 6 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.0476, + "full_recall_at_5": 0.0476, + "mean_recall_at_10": 0.0476, + "full_recall_at_10": 0.0476, + "mean_recall_at_20": 0.0476, + "full_recall_at_20": 0.0476, + "mean_recall_at_1000_tokens": 0.0476, + "mean_recall_at_4000_tokens": 0.0476, + "mean_recall_at_16000_tokens": 0.0476, + "mean_precision_at_10": 0.0159, + "zero_hit_rate": 0.9524, + "median_tokens_delivered": 616, + "median_latency_ms": 869.155, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 2 + }, + "no_path_leak": { + "cases": 20, + "mean_recall_at_5": 0.05, + "full_recall_at_5": 0.05, + "mean_recall_at_10": 0.05, + "full_recall_at_10": 0.05, + "mean_recall_at_20": 0.05, + "full_recall_at_20": 0.05, + "mean_recall_at_1000_tokens": 0.05, + "mean_recall_at_4000_tokens": 0.05, + "mean_recall_at_16000_tokens": 0.05, + "mean_precision_at_10": 0.0167, + "zero_hit_rate": 0.95, + "median_tokens_delivered": 673, + "median_latency_ms": 839.854, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 4 + } + }, + "outcomes": { + "answered": 35, + "indexed-but-returned-nothing": 6 + }, + "resources": { + "baseline_rss_mb": 17152.3, + "peak_rss_mb": 21033.8, + "delta_rss_mb": 3881.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "cocoindex-code", + "summary": { + "all": { + "cases": 41, + "mean_recall_at_5": 0.0976, + "full_recall_at_5": 0.0732, + "mean_recall_at_10": 0.0976, + "full_recall_at_10": 0.0732, + "mean_recall_at_20": 0.0976, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0.0366, + "mean_recall_at_4000_tokens": 0.061, + "mean_recall_at_16000_tokens": 0.0976, + "mean_precision_at_10": 0.0246, + "zero_hit_rate": 0.878, + "median_tokens_delivered": 2607, + "median_latency_ms": 858.794, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 41, + "mean_recall_at_5": 0.0976, + "full_recall_at_5": 0.0732, + "mean_recall_at_10": 0.0976, + "full_recall_at_10": 0.0732, + "mean_recall_at_20": 0.0976, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0.0366, + "mean_recall_at_4000_tokens": 0.061, + "mean_recall_at_16000_tokens": 0.0976, + "mean_precision_at_10": 0.0246, + "zero_hit_rate": 0.878, + "median_tokens_delivered": 2607, + "median_latency_ms": 858.794, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.1429, + "full_recall_at_5": 0.0952, + "mean_recall_at_10": 0.1429, + "full_recall_at_10": 0.0952, + "mean_recall_at_20": 0.1429, + "full_recall_at_20": 0.0952, + "mean_recall_at_1000_tokens": 0.0714, + "mean_recall_at_4000_tokens": 0.0714, + "mean_recall_at_16000_tokens": 0.1429, + "mean_precision_at_10": 0.0413, + "zero_hit_rate": 0.8095, + "median_tokens_delivered": 2607, + "median_latency_ms": 907.846, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 20, + "mean_recall_at_5": 0.05, + "full_recall_at_5": 0.05, + "mean_recall_at_10": 0.05, + "full_recall_at_10": 0.05, + "mean_recall_at_20": 0.05, + "full_recall_at_20": 0.05, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.05, + "mean_recall_at_16000_tokens": 0.05, + "mean_precision_at_10": 0.0071, + "zero_hit_rate": 0.95, + "median_tokens_delivered": 2607, + "median_latency_ms": 835.3835, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 41 + }, + "resources": { + "baseline_rss_mb": 19082.9, + "peak_rss_mb": 25539.5, + "delta_rss_mb": 6456.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "today-little-log-026ffeba899f", + "path_leak": true, + "required_files": [ + "api/auth.ts" + ], + "found": [ + "api/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 18523, + "latency_ms": 27.857, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-0458fb2e26f9", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105308, + "latency_ms": 15.56, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-085a8f98f4e3", + "path_leak": false, + "required_files": [ + "vite.config.ts" + ], + "found": [ + "vite.config.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 109597, + "latency_ms": 14.019, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-0d69b26aa12b", + "path_leak": true, + "required_files": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "found": [], + "missed": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13251, + "latency_ms": 13.689, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-1a06cf12d177", + "path_leak": false, + "required_files": [ + "functions/api/[resource].ts" + ], + "found": [], + "missed": [ + "functions/api/[resource].ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 102343, + "latency_ms": 12.739, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-21e91a0cf7d1", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts" + ], + "found": [], + "missed": [ + "api/_lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19013, + "latency_ms": 13.342, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-2576f31c791b", + "path_leak": true, + "required_files": [ + "eslint.config.js" + ], + "found": [], + "missed": [ + "eslint.config.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20860, + "latency_ms": 15.762, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-311f04610c97", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "api/auth.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19074, + "latency_ms": 12.848, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-3648fa9a9b79", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10438, + "latency_ms": 13.083, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-3822e19e5360", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13637, + "latency_ms": 13.343, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-432012502419", + "path_leak": true, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12447, + "latency_ms": 17.095, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-44e885050a3a", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15916, + "latency_ms": 20.253, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-49d25eea3efb", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13348, + "latency_ms": 12.242, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-4a06c9457671", + "path_leak": true, + "required_files": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "found": [], + "missed": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27482, + "latency_ms": 12.685, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-4c2bf18061a1", + "path_leak": false, + "required_files": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "found": [ + "postcss.config.js" + ], + "missed": [ + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 6, + "tokens_delivered": 14015, + "latency_ms": 13.776, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-5ff55ed07b2f", + "path_leak": true, + "required_files": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17981, + "latency_ms": 13.285, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-6d6487fa6ee3", + "path_leak": false, + "required_files": [ + "src/components/AppLayout.tsx" + ], + "found": [], + "missed": [ + "src/components/AppLayout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123591, + "latency_ms": 14.282, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-6e8bd0230c30", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10493, + "latency_ms": 14.729, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-799f599a32dd", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 104945, + "latency_ms": 15.614, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-7ad692618d92", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123169, + "latency_ms": 17.328, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-83fa639f21f8", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "found": [], + "missed": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 111966, + "latency_ms": 15.998, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-8a71258e7d16", + "path_leak": true, + "required_files": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16346, + "latency_ms": 15.697, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-8b29bbd28725", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19262, + "latency_ms": 15.108, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-93b48d03329b", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10087, + "latency_ms": 16.592, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-9b9d47f48e0b", + "path_leak": false, + "required_files": [ + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12873, + "latency_ms": 16.226, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-9e21586af581", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "found": [ + "src/components/Navbar.tsx" + ], + "missed": [ + ".husky/pre-push", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 10, + "tokens_delivered": 13203, + "latency_ms": 14.46, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-af8c1b0d433b", + "path_leak": false, + "required_files": [ + "api/_schema.ts", + "api/debug.ts" + ], + "found": [], + "missed": [ + "api/_schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 9739, + "latency_ms": 11.713, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-b26636817b1b", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 16299, + "latency_ms": 15.366, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-b297f57a3489", + "path_leak": true, + "required_files": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "found": [ + "src/pages/Index.tsx" + ], + "missed": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Schedule.tsx" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.1429, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1429, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 5, + "tokens_delivered": 13293, + "latency_ms": 14.503, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-bc2139cca1ca", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12829, + "latency_ms": 15.887, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-bd456960c205", + "path_leak": false, + "required_files": [ + "src/components/EntryEditor.tsx" + ], + "found": [], + "missed": [ + "src/components/EntryEditor.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13606, + "latency_ms": 18.372, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-be8f1bcad0c1", + "path_leak": false, + "required_files": [ + ".gitignore" + ], + "found": [], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23062, + "latency_ms": 14.4, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-c1da954218b5", + "path_leak": false, + "required_files": [ + "vitest.config.ts" + ], + "found": [], + "missed": [ + "vitest.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 100717, + "latency_ms": 11.821, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-c1dbe746df7c", + "path_leak": false, + "required_files": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14777, + "latency_ms": 13.673, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-cda10389aa0f", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10942, + "latency_ms": 15.224, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-d3f3c964ec94", + "path_leak": true, + "required_files": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 147928, + "latency_ms": 17.617, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-d543600425f6", + "path_leak": false, + "required_files": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "found": [], + "missed": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13672, + "latency_ms": 15.745, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-dcca049fcd7b", + "path_leak": false, + "required_files": [ + "src/index.css" + ], + "found": [], + "missed": [ + "src/index.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 108092, + "latency_ms": 13.312, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-e2fd6e5d8453", + "path_leak": false, + "required_files": [ + ".saasmaker.json" + ], + "found": [], + "missed": [ + ".saasmaker.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 116794, + "latency_ms": 12.638, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-e37951b10026", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "api/auth.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20848, + "latency_ms": 14.132, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-f81cc54457ab", + "path_leak": false, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18755, + "latency_ms": 14.704, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-026ffeba899f", + "path_leak": true, + "required_files": [ + "api/auth.ts" + ], + "found": [], + "missed": [ + "api/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 217508, + "latency_ms": 19.001, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-0458fb2e26f9", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 250904, + "latency_ms": 20.8, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-085a8f98f4e3", + "path_leak": false, + "required_files": [ + "vite.config.ts" + ], + "found": [], + "missed": [ + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 218109, + "latency_ms": 21.048, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-0d69b26aa12b", + "path_leak": true, + "required_files": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "vendor/saas-maker-ai/package.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 253670, + "latency_ms": 22.518, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-1a06cf12d177", + "path_leak": false, + "required_files": [ + "functions/api/[resource].ts" + ], + "found": [], + "missed": [ + "functions/api/[resource].ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 113820, + "latency_ms": 25.504, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-21e91a0cf7d1", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts" + ], + "found": [ + "api/_lib/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 129831, + "latency_ms": 24.86, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-2576f31c791b", + "path_leak": true, + "required_files": [ + "eslint.config.js" + ], + "found": [], + "missed": [ + "eslint.config.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 121393, + "latency_ms": 25.219, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-311f04610c97", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "api/auth.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 217508, + "latency_ms": 23.691, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-3648fa9a9b79", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 217726, + "latency_ms": 23.724, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-3822e19e5360", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 130907, + "latency_ms": 21.507, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-432012502419", + "path_leak": true, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 217726, + "latency_ms": 22.578, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-44e885050a3a", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 127762, + "latency_ms": 36.254, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-49d25eea3efb", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 130201, + "latency_ms": 22.818, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-4a06c9457671", + "path_leak": true, + "required_files": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "found": [], + "missed": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 131648, + "latency_ms": 23.821, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-4c2bf18061a1", + "path_leak": false, + "required_files": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "found": [ + "vite.config.ts" + ], + "missed": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 161970, + "latency_ms": 26.349, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-5ff55ed07b2f", + "path_leak": true, + "required_files": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 121393, + "latency_ms": 25.01, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-6d6487fa6ee3", + "path_leak": false, + "required_files": [ + "src/components/AppLayout.tsx" + ], + "found": [ + "src/components/AppLayout.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 138033, + "latency_ms": 25.274, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-6e8bd0230c30", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 134327, + "latency_ms": 28.299, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-799f599a32dd", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 130907, + "latency_ms": 23.435, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-7ad692618d92", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 217726, + "latency_ms": 21.13, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-83fa639f21f8", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "found": [ + "api/_lib/auth.ts" + ], + "missed": [ + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 130172, + "latency_ms": 23.265, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-8a71258e7d16", + "path_leak": true, + "required_files": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 219296, + "latency_ms": 22.439, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-8b29bbd28725", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 141649, + "latency_ms": 23.812, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-93b48d03329b", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140248, + "latency_ms": 25.95, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-9b9d47f48e0b", + "path_leak": false, + "required_files": [ + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 134115, + "latency_ms": 23.514, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-9e21586af581", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "found": [ + "src/components/Navbar.tsx" + ], + "missed": [ + ".husky/pre-push", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 114964, + "latency_ms": 39.93, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-af8c1b0d433b", + "path_leak": false, + "required_files": [ + "api/_schema.ts", + "api/debug.ts" + ], + "found": [ + "api/debug.ts" + ], + "missed": [ + "api/_schema.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 129252, + "latency_ms": 22.384, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-b26636817b1b", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 130907, + "latency_ms": 19.753, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-b297f57a3489", + "path_leak": true, + "required_files": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "found": [ + "src/components/AppSidebar.tsx", + "src/pages/Index.tsx" + ], + "missed": [ + "api/profiles.ts", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Schedule.tsx" + ], + "recall_at_5": 0.2857, + "precision_at_5": 0.4, + "recall_at_10": 0.2857, + "precision_at_10": 0.2, + "recall_at_20": 0.2857, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 1, + "tokens_delivered": 255668, + "latency_ms": 33.69, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-bc2139cca1ca", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [ + "tests/prod-auth.spec.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 254337, + "latency_ms": 20.888, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-bd456960c205", + "path_leak": false, + "required_files": [ + "src/components/EntryEditor.tsx" + ], + "found": [], + "missed": [ + "src/components/EntryEditor.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 218528, + "latency_ms": 21.929, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-be8f1bcad0c1", + "path_leak": false, + "required_files": [ + ".gitignore" + ], + "found": [], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 126863, + "latency_ms": 22.948, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-c1da954218b5", + "path_leak": false, + "required_files": [ + "vitest.config.ts" + ], + "found": [], + "missed": [ + "vitest.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140450, + "latency_ms": 37.704, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-c1dbe746df7c", + "path_leak": false, + "required_files": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 129189, + "latency_ms": 21.201, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-cda10389aa0f", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 217718, + "latency_ms": 20.01, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-d3f3c964ec94", + "path_leak": true, + "required_files": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 254316, + "latency_ms": 24.633, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-d543600425f6", + "path_leak": false, + "required_files": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "found": [ + "src/pages/Index.tsx" + ], + "missed": [ + "src/main.tsx" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 138263, + "latency_ms": 25.589, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-dcca049fcd7b", + "path_leak": false, + "required_files": [ + "src/index.css" + ], + "found": [], + "missed": [ + "src/index.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 250904, + "latency_ms": 23.899, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-e2fd6e5d8453", + "path_leak": false, + "required_files": [ + ".saasmaker.json" + ], + "found": [], + "missed": [ + ".saasmaker.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 219008, + "latency_ms": 26.292, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-e37951b10026", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [ + "api/auth.ts", + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 217966, + "latency_ms": 26.823, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-f81cc54457ab", + "path_leak": false, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 253752, + "latency_ms": 24.426, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-026ffeba899f", + "path_leak": true, + "required_files": [ + "api/auth.ts" + ], + "found": [ + "api/auth.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2666, + "latency_ms": 749.872, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-0458fb2e26f9", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3331, + "latency_ms": 754.366, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-085a8f98f4e3", + "path_leak": false, + "required_files": [ + "vite.config.ts" + ], + "found": [], + "missed": [ + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2777, + "latency_ms": 753.229, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-0d69b26aa12b", + "path_leak": true, + "required_files": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "found": [], + "missed": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2993, + "latency_ms": 848.216, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-1a06cf12d177", + "path_leak": false, + "required_files": [ + "functions/api/[resource].ts" + ], + "found": [], + "missed": [ + "functions/api/[resource].ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3593, + "latency_ms": 760.471, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-21e91a0cf7d1", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts" + ], + "found": [ + "api/_lib/auth.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2858, + "latency_ms": 761.402, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-2576f31c791b", + "path_leak": true, + "required_files": [ + "eslint.config.js" + ], + "found": [ + "eslint.config.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3134, + "latency_ms": 852.646, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-311f04610c97", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "api/auth.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3077, + "latency_ms": 704.183, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-3648fa9a9b79", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [ + "scripts/bundle-api.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3705, + "latency_ms": 820.714, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-3822e19e5360", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3106, + "latency_ms": 757.808, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-432012502419", + "path_leak": true, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [ + "scripts/bundle-api.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3557, + "latency_ms": 801.857, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-44e885050a3a", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3314, + "latency_ms": 722.765, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-49d25eea3efb", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3142, + "latency_ms": 788.24, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-4a06c9457671", + "path_leak": true, + "required_files": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "found": [ + "tests/smoke.spec.ts" + ], + "missed": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 10, + "tokens_delivered": 3848, + "latency_ms": 812.214, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-4c2bf18061a1", + "path_leak": false, + "required_files": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "found": [ + "postcss.config.js" + ], + "missed": [ + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1429, + "recall_at_20": 0.25, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 7, + "tokens_delivered": 2989, + "latency_ms": 841.503, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-5ff55ed07b2f", + "path_leak": true, + "required_files": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2970, + "latency_ms": 874.371, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-6d6487fa6ee3", + "path_leak": false, + "required_files": [ + "src/components/AppLayout.tsx" + ], + "found": [], + "missed": [ + "src/components/AppLayout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3798, + "latency_ms": 739.94, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-6e8bd0230c30", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3303, + "latency_ms": 732.262, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-799f599a32dd", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3166, + "latency_ms": 836.394, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-7ad692618d92", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [ + "scripts/bundle-api.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3087, + "latency_ms": 775.457, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-83fa639f21f8", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "found": [ + "api/_lib/auth.ts", + "api/_lib/db.ts" + ], + "missed": [ + "api/_lib/schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3028, + "latency_ms": 895.959, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-8a71258e7d16", + "path_leak": true, + "required_files": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "found": [ + "src/hooks/useChat.ts" + ], + "missed": [ + "src/components/ChatBot.tsx" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3681, + "latency_ms": 855.098, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-8b29bbd28725", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3894, + "latency_ms": 872.577, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-93b48d03329b", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3268, + "latency_ms": 855.269, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-9b9d47f48e0b", + "path_leak": false, + "required_files": [ + "src/hooks/useChat.ts" + ], + "found": [ + "src/hooks/useChat.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3472, + "latency_ms": 762.862, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-9e21586af581", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "found": [], + "missed": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3604, + "latency_ms": 797.391, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-af8c1b0d433b", + "path_leak": false, + "required_files": [ + "api/_schema.ts", + "api/debug.ts" + ], + "found": [ + "api/debug.ts" + ], + "missed": [ + "api/_schema.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 3088, + "latency_ms": 797.37, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-b26636817b1b", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2980, + "latency_ms": 1257.067, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-b297f57a3489", + "path_leak": true, + "required_files": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "found": [ + "api/profiles.ts" + ], + "missed": [ + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1429, + "recall_at_20": 0.1429, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.1429, + "recall_at_4000_tokens": 0.1429, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 1, + "tokens_delivered": 3450, + "latency_ms": 1055.719, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-bc2139cca1ca", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2921, + "latency_ms": 871.866, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-bd456960c205", + "path_leak": false, + "required_files": [ + "src/components/EntryEditor.tsx" + ], + "found": [], + "missed": [ + "src/components/EntryEditor.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3695, + "latency_ms": 722.715, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-be8f1bcad0c1", + "path_leak": false, + "required_files": [ + ".gitignore" + ], + "found": [], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3325, + "latency_ms": 802.772, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-c1da954218b5", + "path_leak": false, + "required_files": [ + "vitest.config.ts" + ], + "found": [ + "vitest.config.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3793, + "latency_ms": 1014.464, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-c1dbe746df7c", + "path_leak": false, + "required_files": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3614, + "latency_ms": 799.127, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-cda10389aa0f", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2918, + "latency_ms": 826.242, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-d3f3c964ec94", + "path_leak": true, + "required_files": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3445, + "latency_ms": 970.171, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-d543600425f6", + "path_leak": false, + "required_files": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "found": [], + "missed": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3956, + "latency_ms": 840.375, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-dcca049fcd7b", + "path_leak": false, + "required_files": [ + "src/index.css" + ], + "found": [], + "missed": [ + "src/index.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3200, + "latency_ms": 821.076, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-e2fd6e5d8453", + "path_leak": false, + "required_files": [ + ".saasmaker.json" + ], + "found": [], + "missed": [ + ".saasmaker.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3197, + "latency_ms": 741.954, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-e37951b10026", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [ + "api/auth.ts" + ], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3192, + "latency_ms": 724.15, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "today-little-log-f81cc54457ab", + "path_leak": false, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2772, + "latency_ms": 1296.604, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-026ffeba899f", + "path_leak": true, + "required_files": [ + "api/auth.ts" + ], + "found": [], + "missed": [ + "api/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 616, + "latency_ms": 662.066, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-0458fb2e26f9", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 891, + "latency_ms": 817.614, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-085a8f98f4e3", + "path_leak": false, + "required_files": [ + "vite.config.ts" + ], + "found": [], + "missed": [ + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 666, + "latency_ms": 760.892, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-0d69b26aa12b", + "path_leak": true, + "required_files": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "found": [], + "missed": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 591, + "latency_ms": 819.922, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-1a06cf12d177", + "path_leak": false, + "required_files": [ + "functions/api/[resource].ts" + ], + "found": [], + "missed": [ + "functions/api/[resource].ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 636, + "latency_ms": 674.589, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-21e91a0cf7d1", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts" + ], + "found": [], + "missed": [ + "api/_lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 585, + "latency_ms": 789.46, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-2576f31c791b", + "path_leak": true, + "required_files": [ + "eslint.config.js" + ], + "found": [], + "missed": [ + "eslint.config.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 595, + "latency_ms": 711.903, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-311f04610c97", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "api/auth.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 616, + "latency_ms": 1093.869, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-3648fa9a9b79", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [ + "scripts/bundle-api.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 420, + "latency_ms": 788.183, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-3822e19e5360", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1634, + "latency_ms": 792.799, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-432012502419", + "path_leak": true, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [ + "scripts/bundle-api.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 431, + "latency_ms": 802.255, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-44e885050a3a", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 784, + "latency_ms": 745.953, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-49d25eea3efb", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 741, + "latency_ms": 767.372, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-4a06c9457671", + "path_leak": true, + "required_files": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "found": [], + "missed": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 870, + "latency_ms": 887.602, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-4c2bf18061a1", + "path_leak": false, + "required_files": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "found": [], + "missed": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2881, + "latency_ms": 489.147, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-5ff55ed07b2f", + "path_leak": true, + "required_files": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 561, + "latency_ms": 798.323, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-6d6487fa6ee3", + "path_leak": false, + "required_files": [ + "src/components/AppLayout.tsx" + ], + "found": [], + "missed": [ + "src/components/AppLayout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 547, + "latency_ms": 459.629, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-6e8bd0230c30", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 577, + "latency_ms": 1021.587, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-799f599a32dd", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 414, + "latency_ms": 739.315, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-7ad692618d92", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 673, + "latency_ms": 797.909, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-83fa639f21f8", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "found": [], + "missed": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 571, + "latency_ms": 866.518, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-8a71258e7d16", + "path_leak": true, + "required_files": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 684, + "latency_ms": 869.998, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-8b29bbd28725", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 695, + "latency_ms": 780.176, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-93b48d03329b", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 502, + "latency_ms": 869.155, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-9b9d47f48e0b", + "path_leak": false, + "required_files": [ + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 481, + "latency_ms": 750.567, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-9e21586af581", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "found": [], + "missed": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 437, + "latency_ms": 870.536, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-af8c1b0d433b", + "path_leak": false, + "required_files": [ + "api/_schema.ts", + "api/debug.ts" + ], + "found": [], + "missed": [ + "api/_schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2511, + "latency_ms": 886.122, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-b26636817b1b", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 633, + "latency_ms": 896.556, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-b297f57a3489", + "path_leak": true, + "required_files": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "found": [], + "missed": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6514, + "latency_ms": 1051.097, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-bc2139cca1ca", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2886, + "latency_ms": 952.277, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-bd456960c205", + "path_leak": false, + "required_files": [ + "src/components/EntryEditor.tsx" + ], + "found": [], + "missed": [ + "src/components/EntryEditor.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1187, + "latency_ms": 883.534, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-be8f1bcad0c1", + "path_leak": false, + "required_files": [ + ".gitignore" + ], + "found": [], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2678, + "latency_ms": 922.553, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-c1da954218b5", + "path_leak": false, + "required_files": [ + "vitest.config.ts" + ], + "found": [], + "missed": [ + "vitest.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 400, + "latency_ms": 969.996, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-c1dbe746df7c", + "path_leak": false, + "required_files": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 673, + "latency_ms": 881.799, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-cda10389aa0f", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1635, + "latency_ms": 1055.965, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-d3f3c964ec94", + "path_leak": true, + "required_files": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 489, + "latency_ms": 1052.552, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-d543600425f6", + "path_leak": false, + "required_files": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "found": [], + "missed": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 520, + "latency_ms": 1529.534, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-dcca049fcd7b", + "path_leak": false, + "required_files": [ + "src/index.css" + ], + "found": [], + "missed": [ + "src/index.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6728, + "latency_ms": 1032.654, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-e2fd6e5d8453", + "path_leak": false, + "required_files": [ + ".saasmaker.json" + ], + "found": [], + "missed": [ + ".saasmaker.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 711, + "latency_ms": 1256.884, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-e37951b10026", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "api/auth.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 616, + "latency_ms": 1083.446, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "today-little-log-f81cc54457ab", + "path_leak": false, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1308, + "latency_ms": 1018.931, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-026ffeba899f", + "path_leak": true, + "required_files": [ + "api/auth.ts" + ], + "found": [], + "missed": [ + "api/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4624, + "latency_ms": 219526.891, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-0458fb2e26f9", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3909, + "latency_ms": 61147.427, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-085a8f98f4e3", + "path_leak": false, + "required_files": [ + "vite.config.ts" + ], + "found": [], + "missed": [ + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5585, + "latency_ms": 15522.098, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-0d69b26aa12b", + "path_leak": true, + "required_files": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "vendor/saas-maker-ai/package.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 4521, + "latency_ms": 12026.906, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-1a06cf12d177", + "path_leak": false, + "required_files": [ + "functions/api/[resource].ts" + ], + "found": [], + "missed": [ + "functions/api/[resource].ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5268, + "latency_ms": 15859.778, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-21e91a0cf7d1", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts" + ], + "found": [], + "missed": [ + "api/_lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4106, + "latency_ms": 125227.026, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-2576f31c791b", + "path_leak": true, + "required_files": [ + "eslint.config.js" + ], + "found": [ + "eslint.config.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4764, + "latency_ms": 112675.632, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-311f04610c97", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [ + "api/auth.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4623, + "latency_ms": 65098.914, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-3648fa9a9b79", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [ + "scripts/bundle-api.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 4444, + "latency_ms": 17512.856, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-3822e19e5360", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4159, + "latency_ms": 6693.608, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-432012502419", + "path_leak": true, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [ + "scripts/bundle-api.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 4773, + "latency_ms": 234063.657, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-44e885050a3a", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1726.943, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-49d25eea3efb", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 912.576, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-4a06c9457671", + "path_leak": true, + "required_files": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "found": [], + "missed": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 547.794, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-4c2bf18061a1", + "path_leak": false, + "required_files": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "found": [], + "missed": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 839.185, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-5ff55ed07b2f", + "path_leak": true, + "required_files": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 759.201, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-6d6487fa6ee3", + "path_leak": false, + "required_files": [ + "src/components/AppLayout.tsx" + ], + "found": [], + "missed": [ + "src/components/AppLayout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 652.384, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-6e8bd0230c30", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 815.097, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-799f599a32dd", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 767.048, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-7ad692618d92", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 759.115, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-83fa639f21f8", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "found": [], + "missed": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 513.53, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-8a71258e7d16", + "path_leak": true, + "required_files": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 773.899, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-8b29bbd28725", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 868.836, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-93b48d03329b", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 907.846, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-9b9d47f48e0b", + "path_leak": false, + "required_files": [ + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 843.648, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-9e21586af581", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "found": [], + "missed": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 649.915, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-af8c1b0d433b", + "path_leak": false, + "required_files": [ + "api/_schema.ts", + "api/debug.ts" + ], + "found": [], + "missed": [ + "api/_schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 941.421, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-b26636817b1b", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 754.521, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-b297f57a3489", + "path_leak": true, + "required_files": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "found": [], + "missed": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 798.715, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-bc2139cca1ca", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 912.261, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-bd456960c205", + "path_leak": false, + "required_files": [ + "src/components/EntryEditor.tsx" + ], + "found": [], + "missed": [ + "src/components/EntryEditor.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 566.317, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-be8f1bcad0c1", + "path_leak": false, + "required_files": [ + ".gitignore" + ], + "found": [], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 904.621, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-c1da954218b5", + "path_leak": false, + "required_files": [ + "vitest.config.ts" + ], + "found": [], + "missed": [ + "vitest.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 752.984, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-c1dbe746df7c", + "path_leak": false, + "required_files": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 732.392, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-cda10389aa0f", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 863.724, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-d3f3c964ec94", + "path_leak": true, + "required_files": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 844.721, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-d543600425f6", + "path_leak": false, + "required_files": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "found": [], + "missed": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 751.151, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-dcca049fcd7b", + "path_leak": false, + "required_files": [ + "src/index.css" + ], + "found": [], + "missed": [ + "src/index.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 731.99, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-e2fd6e5d8453", + "path_leak": false, + "required_files": [ + ".saasmaker.json" + ], + "found": [], + "missed": [ + ".saasmaker.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 831.582, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-e37951b10026", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "api/auth.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 858.794, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "today-little-log-f81cc54457ab", + "path_leak": false, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 918.375, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/P-what-it-takes-to-win.json b/benchmarks/context-retrieval/results/P-what-it-takes-to-win.json new file mode 100644 index 00000000..31460b42 --- /dev/null +++ b/benchmarks/context-retrieval/results/P-what-it-takes-to-win.json @@ -0,0 +1,2735 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "what-it-takes-to-win", + "head": "d3747e8cea4fd1d886ad5c54c09d6096d182352d" + }, + "tier": "small", + "tier_code_files": 65, + "corpus_counts": { + "cases": 14, + "path_leak": 9, + "baseline_blind": 5, + "multi_file": 9 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.4851, + "control": 0, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "Fix birth year estimation: add birth_year column to CSV, recover 195 filtered people", + "revision": "5460fddaeb2ce4baae19ad23051f2ccfa9291390", + "returned": [ + "src/pages/index.astro", + "src/pages/person/[id].astro", + ".github/workflows/ci.yml", + "src/data/archetypes.json", + "src/scripts/merge_research.py" + ] + }, + { + "provider_id": "random-files", + "query": "keep generated profile links audit-safe", + "revision": "25cca210c275ab0311e5c177f5399bab158b4114", + "returned": [ + "src/pages/methodology/index.astro", + "src/pages/insights/index.astro", + "src/pages/compare/index.astro", + "data/research/merge_report.json", + "src/data/archetypes.json" + ] + }, + { + "provider_id": "random-files", + "query": "Fix data quality: remove 6 duplicate IDs, cap 2393 leverage scores at 0-2", + "revision": "77787b7d957c37a1988bd747876d9fd686e5ac58", + "returned": [ + ".github/workflows/ci.yml", + "src/pages/person/[id].astro", + "src/data/archetypes.json", + "src/data/people.json", + "src/scripts/build_founder_engineer_queue.py" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix birth year estimation: add birth_year column to CSV, recover 195 filtered people", + "revision": "5460fddaeb2ce4baae19ad23051f2ccfa9291390", + "returned": [ + "src/data/people.json", + "data/research/merge_report.json", + "src/pages/compare/index.astro", + "src/data/archetypes.json", + "src/pages/person/[id].astro" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix person_id slugs for subagent-researched people", + "revision": "f784628dce33bc8e82d943fdfc2dbd45b26deed9", + "returned": [ + "src/data/people.json", + "data/research/merge_report.json", + ".github/workflows/ci.yml", + "astro.config.mjs", + "package-lock.json" + ] + }, + { + "provider_id": "semble", + "query": "satisfy research script quality checks", + "revision": "1ce2ba36ee066491a27801755037e5f8c68221a6", + "returned": [ + "src/scripts/merge_research.py", + "src/scripts/story-world.ts", + "src/scripts/audit_tiers.mjs", + "src/pages/roi/index.astro", + "src/layouts/Base.astro" + ] + }, + { + "provider_id": "semble", + "query": "keep generated profile links audit-safe", + "revision": "25cca210c275ab0311e5c177f5399bab158b4114", + "returned": [ + "src/scripts/audit_clarity.mjs", + "src/scripts/generate_discovery_surfaces.mjs", + "src/scripts/audit_tiers.mjs", + "data/research/results/gen_researchers_batch_06.py", + "src/pages/insights/index.astro" + ] + }, + { + "provider_id": "semble", + "query": "Fix person_id slugs for subagent-researched people", + "revision": "f784628dce33bc8e82d943fdfc2dbd45b26deed9", + "returned": [ + "src/scripts/merge_research.py", + "src/scripts/filter_candidates.py", + "src/scripts/build_candidate_queue_2025.py", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix birth year estimation: add birth_year column to CSV, recover 195 filtered people", + "revision": "5460fddaeb2ce4baae19ad23051f2ccfa9291390", + "returned": [ + "src/scripts/build_dataset.mjs", + "src/pages/index.astro" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix discovery audit for essay URLs", + "revision": "555e68f0803004424db109cb2dec3d9bec83e420", + "returned": [ + "src/scripts/audit_discovery.mjs", + "src/styles/global.css" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 14, + "mean_recall_at_5": 0.1786, + "full_recall_at_5": 0.0714, + "mean_recall_at_10": 0.2738, + "full_recall_at_10": 0.0714, + "mean_recall_at_20": 0.4167, + "full_recall_at_20": 0.2143, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0357, + "mean_precision_at_10": 0.0643, + "zero_hit_rate": 0.4286, + "median_tokens_delivered": 468773, + "median_latency_ms": 14.2135, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 13, + "mean_recall_at_5": 0.1154, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2179, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3718, + "full_recall_at_20": 0.1538, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0385, + "mean_precision_at_10": 0.0615, + "zero_hit_rate": 0.4615, + "median_tokens_delivered": 280819, + "median_latency_ms": 14.235, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 1, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 6535110, + "median_latency_ms": 12.803, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.1667, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1667, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2778, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0556, + "mean_precision_at_10": 0.0556, + "zero_hit_rate": 0.5556, + "median_tokens_delivered": 255464, + "median_latency_ms": 16.831, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 5, + "mean_recall_at_5": 0.2, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.4667, + "full_recall_at_10": 0.2, + "mean_recall_at_20": 0.6667, + "full_recall_at_20": 0.6, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.08, + "zero_hit_rate": 0.2, + "median_tokens_delivered": 3297240, + "median_latency_ms": 12.967, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 14 + }, + "resources": { + "baseline_rss_mb": 16845.9, + "peak_rss_mb": 16863.8, + "delta_rss_mb": 17.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 14, + "mean_recall_at_5": 0.25, + "full_recall_at_5": 0.0714, + "mean_recall_at_10": 0.4583, + "full_recall_at_10": 0.2857, + "mean_recall_at_20": 0.756, + "full_recall_at_20": 0.5714, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0179, + "mean_precision_at_10": 0.1071, + "zero_hit_rate": 0.1429, + "median_tokens_delivered": 6735249, + "median_latency_ms": 29.527, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 13, + "mean_recall_at_5": 0.1923, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.4167, + "full_recall_at_10": 0.2308, + "mean_recall_at_20": 0.7372, + "full_recall_at_20": 0.5385, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0192, + "mean_precision_at_10": 0.1077, + "zero_hit_rate": 0.1538, + "median_tokens_delivered": 6709755, + "median_latency_ms": 29.768, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 1, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 6760802, + "median_latency_ms": 28.219, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.1667, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.3056, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.6204, + "full_recall_at_20": 0.3333, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0278, + "mean_precision_at_10": 0.0889, + "zero_hit_rate": 0.2222, + "median_tokens_delivered": 6709755, + "median_latency_ms": 29.768, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 5, + "mean_recall_at_5": 0.4, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.7333, + "full_recall_at_10": 0.6, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.14, + "zero_hit_rate": 0, + "median_tokens_delivered": 6760743, + "median_latency_ms": 28.219, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 14 + }, + "resources": { + "baseline_rss_mb": 17128.4, + "peak_rss_mb": 17216.8, + "delta_rss_mb": 88.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 14, + "mean_recall_at_5": 0.4286, + "full_recall_at_5": 0.3571, + "mean_recall_at_10": 0.4613, + "full_recall_at_10": 0.3571, + "mean_recall_at_20": 0.4851, + "full_recall_at_20": 0.3571, + "mean_recall_at_1000_tokens": 0.3691, + "mean_recall_at_4000_tokens": 0.4851, + "mean_recall_at_16000_tokens": 0.4851, + "mean_precision_at_10": 0.0937, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 4827.5, + "median_latency_ms": 1091.859, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 13, + "mean_recall_at_5": 0.3846, + "full_recall_at_5": 0.3077, + "mean_recall_at_10": 0.4199, + "full_recall_at_10": 0.3077, + "mean_recall_at_20": 0.4455, + "full_recall_at_20": 0.3077, + "mean_recall_at_1000_tokens": 0.3205, + "mean_recall_at_4000_tokens": 0.4455, + "mean_recall_at_16000_tokens": 0.4455, + "mean_precision_at_10": 0.0932, + "zero_hit_rate": 0.3077, + "median_tokens_delivered": 4857, + "median_latency_ms": 1083.359, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 1, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 1, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 3977, + "median_latency_ms": 1231.719, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.4444, + "mean_recall_at_10": 0.5139, + "full_recall_at_10": 0.4444, + "mean_recall_at_20": 0.5139, + "full_recall_at_20": 0.4444, + "mean_recall_at_1000_tokens": 0.463, + "mean_recall_at_4000_tokens": 0.5139, + "mean_recall_at_16000_tokens": 0.5139, + "mean_precision_at_10": 0.1123, + "zero_hit_rate": 0.2222, + "median_tokens_delivered": 4857, + "median_latency_ms": 1100.359, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 5, + "mean_recall_at_5": 0.3, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.3667, + "full_recall_at_10": 0.2, + "mean_recall_at_20": 0.4333, + "full_recall_at_20": 0.2, + "mean_recall_at_1000_tokens": 0.2, + "mean_recall_at_4000_tokens": 0.4333, + "mean_recall_at_16000_tokens": 0.4333, + "mean_precision_at_10": 0.06, + "zero_hit_rate": 0.4, + "median_tokens_delivered": 3977, + "median_latency_ms": 1015.14, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 14 + }, + "resources": { + "baseline_rss_mb": 16397.4, + "peak_rss_mb": 17750.1, + "delta_rss_mb": 1352.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "jcodemunch", + "summary": { + "all": { + "cases": 14, + "mean_recall_at_5": 0.2917, + "full_recall_at_5": 0.2143, + "mean_recall_at_10": 0.2917, + "full_recall_at_10": 0.2143, + "mean_recall_at_20": 0.2917, + "full_recall_at_20": 0.2143, + "mean_recall_at_1000_tokens": 0.2917, + "mean_recall_at_4000_tokens": 0.2917, + "mean_recall_at_16000_tokens": 0.2917, + "mean_precision_at_10": 0.2024, + "zero_hit_rate": 0.5714, + "median_tokens_delivered": 514, + "median_latency_ms": 425.715, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 13, + "mean_recall_at_5": 0.2372, + "full_recall_at_5": 0.1538, + "mean_recall_at_10": 0.2372, + "full_recall_at_10": 0.1538, + "mean_recall_at_20": 0.2372, + "full_recall_at_20": 0.1538, + "mean_recall_at_1000_tokens": 0.2372, + "mean_recall_at_4000_tokens": 0.2372, + "mean_recall_at_16000_tokens": 0.2372, + "mean_precision_at_10": 0.1795, + "zero_hit_rate": 0.6154, + "median_tokens_delivered": 514, + "median_latency_ms": 425.752, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 1, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 1, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.5, + "zero_hit_rate": 0, + "median_tokens_delivered": 496, + "median_latency_ms": 368.024, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.3056, + "full_recall_at_5": 0.2222, + "mean_recall_at_10": 0.3056, + "full_recall_at_10": 0.2222, + "mean_recall_at_20": 0.3056, + "full_recall_at_20": 0.2222, + "mean_recall_at_1000_tokens": 0.3056, + "mean_recall_at_4000_tokens": 0.3056, + "mean_recall_at_16000_tokens": 0.3056, + "mean_precision_at_10": 0.2037, + "zero_hit_rate": 0.5556, + "median_tokens_delivered": 520, + "median_latency_ms": 555.682, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 5, + "mean_recall_at_5": 0.2667, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.2667, + "full_recall_at_10": 0.2, + "mean_recall_at_20": 0.2667, + "full_recall_at_20": 0.2, + "mean_recall_at_1000_tokens": 0.2667, + "mean_recall_at_4000_tokens": 0.2667, + "mean_recall_at_16000_tokens": 0.2667, + "mean_precision_at_10": 0.2, + "zero_hit_rate": 0.6, + "median_tokens_delivered": 477, + "median_latency_ms": 410.289, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 14 + }, + "resources": { + "baseline_rss_mb": 17314.5, + "peak_rss_mb": 18358.3, + "delta_rss_mb": 1043.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "cocoindex-code", + "summary": { + "all": { + "cases": 14, + "mean_recall_at_5": 0.1071, + "full_recall_at_5": 0.0714, + "mean_recall_at_10": 0.1071, + "full_recall_at_10": 0.0714, + "mean_recall_at_20": 0.1071, + "full_recall_at_20": 0.0714, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1071, + "mean_recall_at_16000_tokens": 0.1071, + "mean_precision_at_10": 0.0774, + "zero_hit_rate": 0.7857, + "median_tokens_delivered": 2607, + "median_latency_ms": 10474.3585, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 13, + "mean_recall_at_5": 0.1154, + "full_recall_at_5": 0.0769, + "mean_recall_at_10": 0.1154, + "full_recall_at_10": 0.0769, + "mean_recall_at_20": 0.1154, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1154, + "mean_recall_at_16000_tokens": 0.1154, + "mean_precision_at_10": 0.0833, + "zero_hit_rate": 0.7692, + "median_tokens_delivered": 2607, + "median_latency_ms": 10911.559, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 1, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 2552, + "median_latency_ms": 8100.482, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.1667, + "full_recall_at_5": 0.1111, + "mean_recall_at_10": 0.1667, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.1667, + "full_recall_at_20": 0.1111, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1667, + "mean_recall_at_16000_tokens": 0.1667, + "mean_precision_at_10": 0.1204, + "zero_hit_rate": 0.6667, + "median_tokens_delivered": 2607, + "median_latency_ms": 10037.158, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 5, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 2552, + "median_latency_ms": 16965.07, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 14 + }, + "resources": { + "baseline_rss_mb": 17177.3, + "peak_rss_mb": 25561.2, + "delta_rss_mb": 8383.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-3dd1b2f11b56", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 85064, + "latency_ms": 26.734, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-42de39f1e3e7", + "path_leak": true, + "required_files": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "found": [], + "missed": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 146049, + "latency_ms": 21.697, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-62d014f029c5", + "path_leak": true, + "required_files": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/build_dataset.mjs" + ], + "found": [ + "src/data/archetypes.json", + "src/scripts/build_dataset.mjs" + ], + "missed": [ + "src/data/people.csv", + "src/data/people.json" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 116883, + "latency_ms": 14.192, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-6e229e791c6c", + "path_leak": true, + "required_files": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "found": [ + "src/scripts/audit_discovery.mjs" + ], + "missed": [ + "src/scripts/generate_discovery_surfaces.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 70158, + "latency_ms": 13.637, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-77787b7d957c", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 3211331, + "latency_ms": 12.967, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-8105b178afd6", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "found": [ + "src/pages/index.astro" + ], + "missed": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 3297240, + "latency_ms": 14.235, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-9d6998dcea66", + "path_leak": true, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 280819, + "latency_ms": 13.182, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-a6e68b3385a5", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 3474579, + "latency_ms": 12.91, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-c699da18123d", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro" + ], + "found": [ + "src/layouts/Base.astro" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 6535110, + "latency_ms": 12.803, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-c9ac67e7bf08", + "path_leak": true, + "required_files": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "found": [ + "data/research/merge_report.json", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "missed": [ + "src/data/people.csv" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 255464, + "latency_ms": 16.924, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-d3747e8cea4f", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [], + "missed": [ + "src/scripts/marble-world.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1023991, + "latency_ms": 37.752, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-dbe87b68b695", + "path_leak": true, + "required_files": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "found": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.json" + ], + "missed": [ + "src/data/people.csv" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 6487453, + "latency_ms": 13.67, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-ec48ef20dcb2", + "path_leak": true, + "required_files": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "found": [], + "missed": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 193919, + "latency_ms": 16.831, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-eca794d585ba", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [], + "missed": [ + "src/scripts/marble-world.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 656727, + "latency_ms": 19.183, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-3dd1b2f11b56", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 6765015, + "latency_ms": 31.015, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-42de39f1e3e7", + "path_leak": true, + "required_files": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "found": [], + "missed": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10025220, + "latency_ms": 37.024, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-62d014f029c5", + "path_leak": true, + "required_files": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/build_dataset.mjs" + ], + "found": [ + "src/data/archetypes.json", + "src/data/people.json", + "src/scripts/build_dataset.mjs" + ], + "missed": [ + "src/data/people.csv" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 1, + "tokens_delivered": 2687952, + "latency_ms": 26.997, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-6e229e791c6c", + "path_leak": true, + "required_files": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "found": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 6709755, + "latency_ms": 29.768, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-77787b7d957c", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 6427537, + "latency_ms": 25.555, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-8105b178afd6", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "found": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 6760743, + "latency_ms": 35.229, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-9d6998dcea66", + "path_leak": true, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 6709425, + "latency_ms": 25.413, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-a6e68b3385a5", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 3474209, + "latency_ms": 27.375, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-c699da18123d", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro" + ], + "found": [ + "src/layouts/Base.astro" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 6760802, + "latency_ms": 28.219, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-c9ac67e7bf08", + "path_leak": true, + "required_files": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "found": [ + "data/research/merge_report.json", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "missed": [ + "src/data/people.csv" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 255464, + "latency_ms": 20.641, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-d3747e8cea4f", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [ + "src/scripts/marble-world.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 10043630, + "latency_ms": 51.73, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-dbe87b68b695", + "path_leak": true, + "required_files": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "found": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.json" + ], + "missed": [ + "src/data/people.csv" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 1, + "tokens_delivered": 6427548, + "latency_ms": 31.473, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-ec48ef20dcb2", + "path_leak": true, + "required_files": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "found": [ + "src/scripts/audit_clarity.mjs" + ], + "missed": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 10025220, + "latency_ms": 29.286, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-eca794d585ba", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [], + "missed": [ + "src/scripts/marble-world.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10037129, + "latency_ms": 31.8, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-3dd1b2f11b56", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3158, + "latency_ms": 1134.534, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-42de39f1e3e7", + "path_leak": true, + "required_files": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "found": [ + "src/scripts/merge_founder_research.mjs" + ], + "missed": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.125, + "precision_at_10": 0.1, + "recall_at_20": 0.125, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.125, + "recall_at_16000_tokens": 0.125, + "first_hit_rank": 7, + "tokens_delivered": 3229, + "latency_ms": 1664.836, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-62d014f029c5", + "path_leak": true, + "required_files": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/build_dataset.mjs" + ], + "found": [ + "src/scripts/build_dataset.mjs" + ], + "missed": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 4857, + "latency_ms": 880.39, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-6e229e791c6c", + "path_leak": true, + "required_files": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "found": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4294, + "latency_ms": 1100.359, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-77787b7d957c", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [], + "missed": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6041, + "latency_ms": 1015.14, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-8105b178afd6", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "found": [ + "src/layouts/Base.astro", + "src/pages/index.astro" + ], + "missed": [ + "src/pages/explore/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 6, + "tokens_delivered": 3883, + "latency_ms": 993.353, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-9d6998dcea66", + "path_leak": true, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4366, + "latency_ms": 1083.359, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-a6e68b3385a5", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [ + "src/pages/compare/index.astro" + ], + "missed": [ + "src/pages/person/[id].astro" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 7172, + "latency_ms": 795.351, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-c699da18123d", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro" + ], + "found": [ + "src/layouts/Base.astro" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3977, + "latency_ms": 1231.719, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-c9ac67e7bf08", + "path_leak": true, + "required_files": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "found": [ + "src/scripts/merge_research.py" + ], + "missed": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/data/people.json" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1111, + "recall_at_20": 0.25, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 5121, + "latency_ms": 588.366, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-d3747e8cea4f", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [ + "src/scripts/marble-world.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5106, + "latency_ms": 1666.118, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-dbe87b68b695", + "path_leak": true, + "required_files": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "found": [], + "missed": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7872, + "latency_ms": 1008.793, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-ec48ef20dcb2", + "path_leak": true, + "required_files": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "found": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4798, + "latency_ms": 1447.785, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "what-it-takes-to-win-eca794d585ba", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [ + "src/scripts/marble-world.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5444, + "latency_ms": 1470.513, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-3dd1b2f11b56", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 402, + "latency_ms": 425.678, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-42de39f1e3e7", + "path_leak": true, + "required_files": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "found": [], + "missed": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 514, + "latency_ms": 621.35, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-62d014f029c5", + "path_leak": true, + "required_files": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/build_dataset.mjs" + ], + "found": [ + "src/scripts/build_dataset.mjs" + ], + "missed": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.5, + "recall_at_10": 0.25, + "precision_at_10": 0.5, + "recall_at_20": 0.25, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 514, + "latency_ms": 354.405, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-6e229e791c6c", + "path_leak": true, + "required_files": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "found": [ + "src/scripts/audit_discovery.mjs" + ], + "missed": [ + "src/scripts/generate_discovery_surfaces.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 371, + "latency_ms": 517.903, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-77787b7d957c", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [], + "missed": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 472, + "latency_ms": 425.752, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-8105b178afd6", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "found": [ + "src/layouts/Base.astro" + ], + "missed": [ + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 477, + "latency_ms": 389.205, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-9d6998dcea66", + "path_leak": true, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 536, + "latency_ms": 555.682, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-a6e68b3385a5", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [], + "missed": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 575, + "latency_ms": 410.289, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-c699da18123d", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro" + ], + "found": [ + "src/layouts/Base.astro" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 496, + "latency_ms": 368.024, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-c9ac67e7bf08", + "path_leak": true, + "required_files": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "found": [], + "missed": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 549, + "latency_ms": 360.029, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-d3747e8cea4f", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [ + "src/scripts/marble-world.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 514, + "latency_ms": 659.655, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-dbe87b68b695", + "path_leak": true, + "required_files": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "found": [], + "missed": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 520, + "latency_ms": 372.409, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-ec48ef20dcb2", + "path_leak": true, + "required_files": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "found": [], + "missed": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 635, + "latency_ms": 574.21, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "what-it-takes-to-win-eca794d585ba", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [ + "src/scripts/marble-world.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 565, + "latency_ms": 609.748, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-3dd1b2f11b56", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3215, + "latency_ms": 5532.777, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-42de39f1e3e7", + "path_leak": true, + "required_files": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "found": [], + "missed": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2823, + "latency_ms": 10037.158, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-62d014f029c5", + "path_leak": true, + "required_files": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/build_dataset.mjs" + ], + "found": [ + "src/data/people.json" + ], + "missed": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/scripts/build_dataset.mjs" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.25, + "recall_at_10": 0.25, + "precision_at_10": 0.25, + "recall_at_20": 0.25, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 3149, + "latency_ms": 12256.245, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-6e229e791c6c", + "path_leak": true, + "required_files": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "found": [], + "missed": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2593, + "latency_ms": 12269.09, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-77787b7d957c", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [], + "missed": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3499, + "latency_ms": 33441.049, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-8105b178afd6", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "found": [], + "missed": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2467, + "latency_ms": 16965.07, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-9d6998dcea66", + "path_leak": true, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2976, + "latency_ms": 21452.446, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-a6e68b3385a5", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [], + "missed": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2478, + "latency_ms": 17881.154, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-c699da18123d", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro" + ], + "found": [], + "missed": [ + "src/layouts/Base.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2552, + "latency_ms": 8100.482, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-c9ac67e7bf08", + "path_leak": true, + "required_files": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "found": [ + "src/data/people.json" + ], + "missed": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/scripts/merge_research.py" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.5, + "recall_at_10": 0.25, + "precision_at_10": 0.5, + "recall_at_20": 0.25, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 2977, + "latency_ms": 10911.559, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-d3747e8cea4f", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [], + "missed": [ + "src/scripts/marble-world.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 4279.886, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-dbe87b68b695", + "path_leak": true, + "required_files": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "found": [], + "missed": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 833.02, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-ec48ef20dcb2", + "path_leak": true, + "required_files": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "found": [], + "missed": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1122.786, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "what-it-takes-to-win-eca794d585ba", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [], + "missed": [ + "src/scripts/marble-world.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2607, + "latency_ms": 1099.788, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/RW-express.json b/benchmarks/context-retrieval/results/RW-express.json new file mode 100644 index 00000000..33b3ee61 --- /dev/null +++ b/benchmarks/context-retrieval/results/RW-express.json @@ -0,0 +1,2688 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "express", + "head": "a3714473feb3d2908add734d340e7755fd85e0a3" + }, + "tier": "small", + "tier_code_files": 141, + "corpus_counts": { + "cases": 28, + "path_leak": 9, + "baseline_blind": 19, + "multi_file": 14 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": false, + "best": 0.3036, + "control": 0.2143, + "reason": "a query-blind control scored 21.4% against a leader's 30.4%. Suspect the metric, not the tools." + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "Fix hanging on large stack of sync routes", + "revision": "75e0c7a2c91665f44d053d83be15f8ecd0177f41", + "returned": [ + "test/req.range.js", + "test/acceptance/route-separation.js", + "test/res.append.js", + "test/req.accepts.js", + "lib/router/layer.js" + ] + }, + { + "provider_id": "random-files", + "query": "remove duplicate word (#6456)", + "revision": "5da5a11a498a3034623960861e542a3b39b00c94", + "returned": [ + "test/res.format.js", + "test/res.vary.js", + "test/res.cookie.js", + "lib/utils.js", + "examples/view-constructor/github-view.js" + ] + }, + { + "provider_id": "churn-ranked", + "query": "add Content-Length header only if Transfer-Encoding is not present (#4893)", + "revision": "59e205a57a04fced6bb7b8ec0b5dec29461a9996", + "returned": [ + "package.json", + ".github/workflows/ci.yml", + "lib/response.js", + ".github/workflows/legacy.yml", + ".github/workflows/scorecard.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix handling very large stacks of sync middleware", + "revision": "92c5ce59f51cce4b3598fd040117772fac42dce8", + "returned": [ + "package.json", + "appveyor.yml", + "lib/response.js", + "test/res.sendFile.js", + ".github/workflows/ci.yml" + ] + }, + { + "provider_id": "repowise", + "query": "add Content-Length header only if Transfer-Encoding is not present (#4893)", + "revision": "59e205a57a04fced6bb7b8ec0b5dec29461a9996", + "returned": [ + "lib/response.js", + "examples/online/index.js", + "lib/application.js", + "test/support/utils.js", + "examples/web-service/index.js" + ] + }, + { + "provider_id": "repowise", + "query": "Fix handling very large stacks of sync middleware", + "revision": "92c5ce59f51cce4b3598fd040117772fac42dce8", + "returned": [ + "examples/error/index.js", + "lib/view.js", + "examples/route-middleware/index.js", + "examples/view-locals/index.js", + "lib/express.js" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.0179, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0357, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0804, + "full_recall_at_20": 0.0357, + "mean_recall_at_1000_tokens": 0.0179, + "mean_recall_at_4000_tokens": 0.0179, + "mean_recall_at_16000_tokens": 0.0714, + "mean_precision_at_10": 0.0071, + "zero_hit_rate": 0.8571, + "median_tokens_delivered": 17557.5, + "median_latency_ms": 12.606, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 28, + "mean_recall_at_5": 0.0179, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0357, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0804, + "full_recall_at_20": 0.0357, + "mean_recall_at_1000_tokens": 0.0179, + "mean_recall_at_4000_tokens": 0.0179, + "mean_recall_at_16000_tokens": 0.0714, + "mean_precision_at_10": 0.0071, + "zero_hit_rate": 0.8571, + "median_tokens_delivered": 17557.5, + "median_latency_ms": 12.606, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 17126, + "median_latency_ms": 13.029, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.0263, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0526, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1184, + "full_recall_at_20": 0.0526, + "mean_recall_at_1000_tokens": 0.0263, + "mean_recall_at_4000_tokens": 0.0263, + "mean_recall_at_16000_tokens": 0.1053, + "mean_precision_at_10": 0.0105, + "zero_hit_rate": 0.7895, + "median_tokens_delivered": 17991, + "median_latency_ms": 12.563, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 28 + }, + "resources": { + "baseline_rss_mb": 21696.6, + "peak_rss_mb": 21732.5, + "delta_rss_mb": 35.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.4345, + "full_recall_at_5": 0.2857, + "mean_recall_at_10": 0.5625, + "full_recall_at_10": 0.3929, + "mean_recall_at_20": 0.619, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2143, + "mean_recall_at_16000_tokens": 0.4881, + "mean_precision_at_10": 0.0857, + "zero_hit_rate": 0.25, + "median_tokens_delivered": 55061.5, + "median_latency_ms": 23.014, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 28, + "mean_recall_at_5": 0.4345, + "full_recall_at_5": 0.2857, + "mean_recall_at_10": 0.5625, + "full_recall_at_10": 0.3929, + "mean_recall_at_20": 0.619, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2143, + "mean_recall_at_16000_tokens": 0.4881, + "mean_precision_at_10": 0.0857, + "zero_hit_rate": 0.25, + "median_tokens_delivered": 55061.5, + "median_latency_ms": 23.014, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.3148, + "full_recall_at_5": 0.1111, + "mean_recall_at_10": 0.3148, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.3148, + "full_recall_at_20": 0.1111, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.3148, + "mean_precision_at_10": 0.0556, + "zero_hit_rate": 0.4444, + "median_tokens_delivered": 54998, + "median_latency_ms": 23.63, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.4912, + "full_recall_at_5": 0.3684, + "mean_recall_at_10": 0.6798, + "full_recall_at_10": 0.5263, + "mean_recall_at_20": 0.7632, + "full_recall_at_20": 0.6842, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.3158, + "mean_recall_at_16000_tokens": 0.5702, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.1579, + "median_tokens_delivered": 55125, + "median_latency_ms": 22.829, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 28 + }, + "resources": { + "baseline_rss_mb": 21764.2, + "peak_rss_mb": 21813.4, + "delta_rss_mb": 49.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "repowise", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.1905, + "full_recall_at_5": 0.0714, + "mean_recall_at_10": 0.3036, + "full_recall_at_10": 0.1071, + "mean_recall_at_20": 0.3036, + "full_recall_at_20": 0.1071, + "mean_recall_at_1000_tokens": 0.2351, + "mean_recall_at_4000_tokens": 0.3036, + "mean_recall_at_16000_tokens": 0.3036, + "mean_precision_at_10": 0.0611, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 1706.5, + "median_latency_ms": 37171.323, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 28, + "mean_recall_at_5": 0.1905, + "full_recall_at_5": 0.0714, + "mean_recall_at_10": 0.3036, + "full_recall_at_10": 0.1071, + "mean_recall_at_20": 0.3036, + "full_recall_at_20": 0.1071, + "mean_recall_at_1000_tokens": 0.2351, + "mean_recall_at_4000_tokens": 0.3036, + "mean_recall_at_16000_tokens": 0.3036, + "mean_precision_at_10": 0.0611, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 1706.5, + "median_latency_ms": 37171.323, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.2593, + "full_recall_at_5": 0.1111, + "mean_recall_at_10": 0.3704, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.3704, + "full_recall_at_20": 0.1111, + "mean_recall_at_1000_tokens": 0.3148, + "mean_recall_at_4000_tokens": 0.3704, + "mean_recall_at_16000_tokens": 0.3704, + "mean_precision_at_10": 0.0667, + "zero_hit_rate": 0.3333, + "median_tokens_delivered": 2169, + "median_latency_ms": 37179.759, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.1579, + "full_recall_at_5": 0.0526, + "mean_recall_at_10": 0.2719, + "full_recall_at_10": 0.1053, + "mean_recall_at_20": 0.2719, + "full_recall_at_20": 0.1053, + "mean_recall_at_1000_tokens": 0.1974, + "mean_recall_at_4000_tokens": 0.2719, + "mean_recall_at_16000_tokens": 0.2719, + "mean_precision_at_10": 0.0585, + "zero_hit_rate": 0.5789, + "median_tokens_delivered": 1521, + "median_latency_ms": 37162.887, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 28 + }, + "resources": { + "baseline_rss_mb": 21839.1, + "peak_rss_mb": 24519, + "delta_rss_mb": 2680, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: a query-blind control scored 21.4% against a leader's 30.4%. Suspect the metric, not the tools.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15887, + "latency_ms": 25.982, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.send.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24167, + "latency_ms": 12.244, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [], + "missed": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17991, + "latency_ms": 12.386, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15115, + "latency_ms": 12.98, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8352, + "latency_ms": 12.251, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25943, + "latency_ms": 12.535, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24105, + "latency_ms": 12.42, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.redirect.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17989, + "latency_ms": 13.36, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.cookie.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10199, + "latency_ms": 13.224, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25824, + "latency_ms": 12.3, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [ + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 21567, + "latency_ms": 12.255, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14979, + "latency_ms": 12.059, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21348, + "latency_ms": 12.312, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28836, + "latency_ms": 12.318, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "test/Route.js" + ], + "missed": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 25626, + "latency_ms": 12.146, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26880, + "latency_ms": 12.572, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12752, + "latency_ms": 12.64, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12150, + "latency_ms": 12.563, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [], + "missed": [ + "lib/router/index.js", + "test/app.router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12313, + "latency_ms": 12.863, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14066, + "latency_ms": 12.784, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24966, + "latency_ms": 13.234, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [], + "missed": [ + "lib/response.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18218, + "latency_ms": 14.491, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 22036, + "latency_ms": 12.755, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [], + "missed": [ + "examples/search/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17126, + "latency_ms": 12.417, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15153, + "latency_ms": 13.131, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [], + "missed": [ + "lib/router/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15550, + "latency_ms": 13.029, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [ + "test/res.format.js" + ], + "missed": [ + "test/app.router.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 15695, + "latency_ms": 13.005, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10590, + "latency_ms": 12.77, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 48438, + "latency_ms": 25.438, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [ + "lib/response.js", + "test/res.send.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 63857, + "latency_ms": 23.199, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 68922, + "latency_ms": 23.409, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 46929, + "latency_ms": 22.829, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 44158, + "latency_ms": 21.399, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 69122, + "latency_ms": 21.888, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48438, + "latency_ms": 24.443, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.redirect.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 56208, + "latency_ms": 25.025, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.cookie.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 54109, + "latency_ms": 23.959, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69044, + "latency_ms": 22.344, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 57263, + "latency_ms": 24.394, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 48439, + "latency_ms": 24.053, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68691, + "latency_ms": 23.813, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68940, + "latency_ms": 21.569, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 57070, + "latency_ms": 21.894, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [ + "test/Router.js" + ], + "missed": [ + "lib/router/route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 55125, + "latency_ms": 21.455, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54998, + "latency_ms": 21.195, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 63660, + "latency_ms": 23.345, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [ + "lib/router/index.js", + "test/app.router.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 46930, + "latency_ms": 21.497, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63835, + "latency_ms": 23.28, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.jsonp.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 47132, + "latency_ms": 22.054, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 46929, + "latency_ms": 21.226, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 52216, + "latency_ms": 24.426, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [], + "missed": [ + "examples/search/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63752, + "latency_ms": 25.16, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.json.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 47164, + "latency_ms": 23.63, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 44146, + "latency_ms": 20.388, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [ + "test/app.router.js" + ], + "missed": [ + "test/res.format.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 69122, + "latency_ms": 21.805, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 52218, + "latency_ms": 21.611, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1414, + "latency_ms": 37625.567, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.send.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1571, + "latency_ms": 38183.3, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/application.js", + "lib/response.js" + ], + "missed": [ + "test/res.cookie.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 6, + "tokens_delivered": 2034, + "latency_ms": 36968.61, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2275, + "latency_ms": 37162.887, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.sendFile.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2528, + "latency_ms": 36701.971, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1817, + "latency_ms": 37801.875, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1177, + "latency_ms": 38410.176, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.redirect.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1661, + "latency_ms": 38492.106, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.cookie.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 2471, + "latency_ms": 37196.995, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2207, + "latency_ms": 37818.108, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1127, + "latency_ms": 38015.737, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1200, + "latency_ms": 37192.458, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/req.acceptsCharsets.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1622, + "latency_ms": 36968.574, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1752, + "latency_ms": 36691.28, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "lib/router/route.js" + ], + "missed": [ + "test/Route.js", + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1350, + "latency_ms": 36571.607, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [ + "lib/router/route.js" + ], + "missed": [ + "test/Router.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1521, + "latency_ms": 36692.728, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [ + "lib/router/route.js" + ], + "missed": [ + "test/Route.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1810, + "latency_ms": 37179.759, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 358, + "latency_ms": 36875.206, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [ + "test/app.router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1762, + "latency_ms": 37306.791, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 550, + "latency_ms": 37419.626, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.jsonp.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2480, + "latency_ms": 38056.783, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2310, + "latency_ms": 37850.907, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1911, + "latency_ms": 36884.64, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [], + "missed": [ + "examples/search/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2331, + "latency_ms": 36371.808, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.json.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 1347, + "latency_ms": 36766.261, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2169, + "latency_ms": 35855.624, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [], + "missed": [ + "test/app.router.js", + "test/res.format.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 912, + "latency_ms": 36084.698, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1158, + "latency_ms": 36719.118, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/RW-flask.json b/benchmarks/context-retrieval/results/RW-flask.json new file mode 100644 index 00000000..ccffaf08 --- /dev/null +++ b/benchmarks/context-retrieval/results/RW-flask.json @@ -0,0 +1,3585 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "flask", + "head": "d318b683471101618febed18996405ad26462110" + }, + "tier": "small", + "tier_code_files": 83, + "corpus_counts": { + "cases": 39, + "path_leak": 7, + "baseline_blind": 32, + "multi_file": 17 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.2628, + "control": 0.0321, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "fix uninstalled package tests under tox", + "revision": "88bcf78439b66b5fcba9f4d3a1c56307f98dbf1d", + "returned": [ + "examples/tutorial/tests/test_factory.py", + "tests/test_apps/helloworld/wsgi.py", + "tests/test_apps/subdomaintestmodule/__init__.py", + "tests/test_reqctx.py", + "docs/conf.py" + ] + }, + { + "provider_id": "random-files", + "query": "fix subdomain_matching=False behavior", + "revision": "07c7d5730a2685ef2281cc635e289685e5c3d478", + "returned": [ + ".readthedocs.yaml", + "tests/test_views.py", + "src/flask/cli.py", + "tests/test_apps/cliapp/inner1/__init__.py", + "src/flask/config.py" + ] + }, + { + "provider_id": "random-files", + "query": "fix provide_automatic_options override", + "revision": "d3b78fd18a8d9e224cb9ef58a23cec9b1ffc9ce9", + "returned": [ + "tests/test_async.py", + "tests/test_views.py", + "tests/conftest.py", + "tests/test_apps/cliapp/app.py", + "tests/type_check/typing_app_decorators.py" + ] + }, + { + "provider_id": "churn-ranked", + "query": "fix importing Markup from flask", + "revision": "345f18442ceb29f9e365af99fc85bdc5986323d2", + "returned": [ + "src/flask/app.py", + ".pre-commit-config.yaml", + "src/flask/helpers.py", + "src/flask/blueprints.py", + "src/flask/__init__.py" + ] + }, + { + "provider_id": "churn-ranked", + "query": "fix pytest 7 warnings", + "revision": "6f7d99ce4b7dc2bad52683e64fec7008dd0b580f", + "returned": [ + "src/flask/app.py", + "src/flask/helpers.py", + "src/flask/blueprints.py", + "src/flask/cli.py", + "tests/test_basic.py" + ] + }, + { + "provider_id": "churn-ranked", + "query": "fix annotation for json.loads", + "revision": "69f71b4d94006678a2463f76a4504208d195c367", + "returned": [ + "src/flask/app.py", + "src/flask/helpers.py", + ".pre-commit-config.yaml", + "src/flask/blueprints.py", + "src/flask/cli.py" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.0427, + "full_recall_at_5": 0.0256, + "mean_recall_at_10": 0.0962, + "full_recall_at_10": 0.0513, + "mean_recall_at_20": 0.1859, + "full_recall_at_20": 0.1026, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0171, + "mean_recall_at_16000_tokens": 0.094, + "mean_precision_at_10": 0.0179, + "zero_hit_rate": 0.6667, + "median_tokens_delivered": 30503, + "median_latency_ms": 12.535, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 37, + "mean_recall_at_5": 0.018, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0473, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1419, + "full_recall_at_20": 0.0541, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.018, + "mean_recall_at_16000_tokens": 0.045, + "mean_precision_at_10": 0.0135, + "zero_hit_rate": 0.7027, + "median_tokens_delivered": 30706, + "median_latency_ms": 12.46, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 26740, + "median_latency_ms": 12.868, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0476, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.119, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0714, + "mean_precision_at_10": 0.0143, + "zero_hit_rate": 0.7143, + "median_tokens_delivered": 26267, + "median_latency_ms": 12.535, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.0521, + "full_recall_at_5": 0.0313, + "mean_recall_at_10": 0.1068, + "full_recall_at_10": 0.0625, + "mean_recall_at_20": 0.2005, + "full_recall_at_20": 0.125, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0208, + "mean_recall_at_16000_tokens": 0.099, + "mean_precision_at_10": 0.0188, + "zero_hit_rate": 0.6563, + "median_tokens_delivered": 31713, + "median_latency_ms": 12.5065, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 39 + }, + "resources": { + "baseline_rss_mb": 21697.3, + "peak_rss_mb": 21729.1, + "delta_rss_mb": 31.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.2671, + "full_recall_at_5": 0.1795, + "mean_recall_at_10": 0.4573, + "full_recall_at_10": 0.3077, + "mean_recall_at_20": 0.6026, + "full_recall_at_20": 0.4359, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0321, + "mean_recall_at_16000_tokens": 0.1603, + "mean_precision_at_10": 0.0718, + "zero_hit_rate": 0.2308, + "median_tokens_delivered": 80328, + "median_latency_ms": 25.947, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 37, + "mean_recall_at_5": 0.2545, + "full_recall_at_5": 0.1622, + "mean_recall_at_10": 0.455, + "full_recall_at_10": 0.2973, + "mean_recall_at_20": 0.6081, + "full_recall_at_20": 0.4324, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0338, + "mean_recall_at_16000_tokens": 0.1419, + "mean_precision_at_10": 0.073, + "zero_hit_rate": 0.2162, + "median_tokens_delivered": 80328, + "median_latency_ms": 25.947, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.5, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.5, + "mean_precision_at_10": 0.05, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 89733, + "median_latency_ms": 25.835, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0.1429, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.4048, + "full_recall_at_10": 0.1429, + "mean_recall_at_20": 0.4762, + "full_recall_at_20": 0.2857, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0714, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 98043, + "median_latency_ms": 25.833, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.2943, + "full_recall_at_5": 0.2188, + "mean_recall_at_10": 0.4688, + "full_recall_at_10": 0.3438, + "mean_recall_at_20": 0.6302, + "full_recall_at_20": 0.4688, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0391, + "mean_recall_at_16000_tokens": 0.1953, + "mean_precision_at_10": 0.0719, + "zero_hit_rate": 0.2188, + "median_tokens_delivered": 80264.5, + "median_latency_ms": 25.9485, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 39 + }, + "resources": { + "baseline_rss_mb": 21773.6, + "peak_rss_mb": 21873.7, + "delta_rss_mb": 100.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "repowise", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.2073, + "full_recall_at_5": 0.1282, + "mean_recall_at_10": 0.2628, + "full_recall_at_10": 0.1282, + "mean_recall_at_20": 0.2628, + "full_recall_at_20": 0.1282, + "mean_recall_at_1000_tokens": 0.1688, + "mean_recall_at_4000_tokens": 0.2628, + "mean_recall_at_16000_tokens": 0.2628, + "mean_precision_at_10": 0.0601, + "zero_hit_rate": 0.5641, + "median_tokens_delivered": 1500, + "median_latency_ms": 36930.96, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 37, + "mean_recall_at_5": 0.2185, + "full_recall_at_5": 0.1351, + "mean_recall_at_10": 0.277, + "full_recall_at_10": 0.1351, + "mean_recall_at_20": 0.277, + "full_recall_at_20": 0.1351, + "mean_recall_at_1000_tokens": 0.1779, + "mean_recall_at_4000_tokens": 0.277, + "mean_recall_at_16000_tokens": 0.277, + "mean_precision_at_10": 0.0634, + "zero_hit_rate": 0.5405, + "median_tokens_delivered": 1500, + "median_latency_ms": 37010.679, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 1782.5, + "median_latency_ms": 36736.3, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0.2619, + "full_recall_at_5": 0.1429, + "mean_recall_at_10": 0.3333, + "full_recall_at_10": 0.1429, + "mean_recall_at_20": 0.3333, + "full_recall_at_20": 0.1429, + "mean_recall_at_1000_tokens": 0.119, + "mean_recall_at_4000_tokens": 0.3333, + "mean_recall_at_16000_tokens": 0.3333, + "mean_precision_at_10": 0.1052, + "zero_hit_rate": 0.4286, + "median_tokens_delivered": 1813, + "median_latency_ms": 37208.073, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.1953, + "full_recall_at_5": 0.125, + "mean_recall_at_10": 0.2474, + "full_recall_at_10": 0.125, + "mean_recall_at_20": 0.2474, + "full_recall_at_20": 0.125, + "mean_recall_at_1000_tokens": 0.1797, + "mean_recall_at_4000_tokens": 0.2474, + "mean_recall_at_16000_tokens": 0.2474, + "mean_precision_at_10": 0.0503, + "zero_hit_rate": 0.5938, + "median_tokens_delivered": 1278.5, + "median_latency_ms": 36926.7155, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 39 + }, + "resources": { + "baseline_rss_mb": 21902.6, + "peak_rss_mb": 24493.2, + "delta_rss_mb": 2590.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17705, + "latency_ms": 26.093, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [], + "missed": [ + "src/flask/ctx.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22208, + "latency_ms": 12.963, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30144, + "latency_ms": 12.158, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30503, + "latency_ms": 12.338, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 11009, + "latency_ms": 12.37, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27983, + "latency_ms": 12.601, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "tests/test_cli.py" + ], + "missed": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 26044, + "latency_ms": 12.292, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40575, + "latency_ms": 13.743, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_appctx.py" + ], + "missed": [ + ".flake8", + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 37624, + "latency_ms": 12.896, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py" + ], + "missed": [ + "tests/test_instance_config.py", + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 62976, + "latency_ms": 12.224, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36940, + "latency_ms": 12.342, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_reqctx.py" + ], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 18661, + "latency_ms": 12.185, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 50380, + "latency_ms": 12.194, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30706, + "latency_ms": 12.278, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 45986, + "latency_ms": 12.176, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 32720, + "latency_ms": 12.553, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [ + "src/flask/config.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 28009, + "latency_ms": 12.64, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36384, + "latency_ms": 12.46, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [], + "missed": [ + "src/flask/typing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30334, + "latency_ms": 12.963, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28572, + "latency_ms": 12.74, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43383, + "latency_ms": 13.254, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19148, + "latency_ms": 14.507, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [], + "missed": [ + "pyproject.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33010, + "latency_ms": 12.792, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25252, + "latency_ms": 12.453, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [ + ".pre-commit-config.yaml" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 25471, + "latency_ms": 13.096, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [], + "missed": [ + "src/flask/wrappers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45386, + "latency_ms": 13.038, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23746, + "latency_ms": 12.847, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [ + "CHANGES.rst", + "src/flask/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 36859, + "latency_ms": 12.784, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 36829, + "latency_ms": 13.213, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51711, + "latency_ms": 13.765, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "tests/test_blueprints.py" + ], + "missed": [ + "src/flask/blueprints.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 26267, + "latency_ms": 12.535, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [], + "missed": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23985, + "latency_ms": 11.898, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [], + "missed": [ + "examples/javascript/tests/test_js_example.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40972, + "latency_ms": 13.029, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41679, + "latency_ms": 11.557, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35855, + "latency_ms": 11.774, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "tests/test_views.py" + ], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 2, + "tokens_delivered": 27943, + "latency_ms": 12.047, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19505, + "latency_ms": 10.987, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42482, + "latency_ms": 11.881, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25042, + "latency_ms": 10.872, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 98958, + "latency_ms": 27.022, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 98182, + "latency_ms": 27.819, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 97356, + "latency_ms": 27.533, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 100544, + "latency_ms": 24.224, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [ + "src/flask/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 97421, + "latency_ms": 28.761, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [ + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 70486, + "latency_ms": 26.876, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "src/flask/cli.py", + "tests/test_cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 70069, + "latency_ms": 24.416, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 76284, + "latency_ms": 24.839, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + ".flake8", + "tests/test_appctx.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 101135, + "latency_ms": 24.004, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py" + ], + "missed": [ + "tests/test_instance_config.py", + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 98043, + "latency_ms": 24.401, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70565, + "latency_ms": 26.289, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "tests/test_reqctx.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 100513, + "latency_ms": 27.589, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 72596, + "latency_ms": 25.654, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [ + "src/flask/testing.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 73072, + "latency_ms": 25.551, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 96806, + "latency_ms": 24.692, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 81351, + "latency_ms": 28.366, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [], + "missed": [ + "src/flask/config.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 99251, + "latency_ms": 26.214, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70565, + "latency_ms": 24.265, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [ + "src/flask/typing.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 75828, + "latency_ms": 24.8, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 96787, + "latency_ms": 24.309, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 70795, + "latency_ms": 25.969, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 97421, + "latency_ms": 27.385, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [ + "pyproject.toml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 80328, + "latency_ms": 26.558, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 70553, + "latency_ms": 25.95, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [ + ".pre-commit-config.yaml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 80215, + "latency_ms": 25.456, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [], + "missed": [ + "src/flask/wrappers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70730, + "latency_ms": 27.898, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 99308, + "latency_ms": 24.672, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 97261, + "latency_ms": 25.314, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 95560, + "latency_ms": 25.238, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [ + "src/flask/sessions.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 70731, + "latency_ms": 24.71, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 100533, + "latency_ms": 25.833, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 81260, + "latency_ms": 28.229, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [], + "missed": [ + "examples/javascript/tests/test_js_example.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70553, + "latency_ms": 26.218, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 71520, + "latency_ms": 25.524, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70565, + "latency_ms": 26.302, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_cli.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py", + "tests/test_views.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 80314, + "latency_ms": 27.516, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [ + "src/flask/sessions.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 70461, + "latency_ms": 28.469, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70069, + "latency_ms": 25.947, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [ + "src/flask/json/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 98082, + "latency_ms": 24.925, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1863, + "latency_ms": 37522.289, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 597, + "latency_ms": 37826.041, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 871, + "latency_ms": 37181.477, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1902, + "latency_ms": 36930.96, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [ + "src/flask/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1066, + "latency_ms": 37322.468, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 512, + "latency_ms": 37797.71, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_cli.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 7, + "tokens_delivered": 1698, + "latency_ms": 38189.935, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 1332, + "latency_ms": 38042.44, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [], + "missed": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1500, + "latency_ms": 37514.078, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py" + ], + "missed": [ + "tests/test_instance_config.py", + "tox.ini" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.125, + "recall_at_20": 0.3333, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 1840, + "latency_ms": 37320.603, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2474, + "latency_ms": 38132.384, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1225, + "latency_ms": 37066.241, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 1515, + "latency_ms": 37133.837, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [ + "src/flask/testing.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1889, + "latency_ms": 36859.157, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2304, + "latency_ms": 36171.952, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1875, + "latency_ms": 36647.643, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [], + "missed": [ + "src/flask/config.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3138, + "latency_ms": 36815.407, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2320, + "latency_ms": 37010.679, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [ + "src/flask/typing.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1813, + "latency_ms": 37208.073, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 427, + "latency_ms": 37447.264, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 512, + "latency_ms": 37891.395, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 512, + "latency_ms": 37466.778, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [], + "missed": [ + "pyproject.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1759, + "latency_ms": 36922.471, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 512, + "latency_ms": 36371.651, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [], + "missed": [ + ".pre-commit-config.yaml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 427, + "latency_ms": 36657.193, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [ + "src/flask/wrappers.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1507, + "latency_ms": 36614.271, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1956, + "latency_ms": 35968.193, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 8, + "tokens_delivered": 2313, + "latency_ms": 37064.403, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 427, + "latency_ms": 36218.138, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 512, + "latency_ms": 35483.084, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/blueprints.py" + ], + "missed": [ + "tests/test_blueprints.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1555, + "latency_ms": 35302.486, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [], + "missed": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 512, + "latency_ms": 35249.011, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [], + "missed": [ + "examples/javascript/tests/test_js_example.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2080, + "latency_ms": 35082.35, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 512, + "latency_ms": 35391.763, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 427, + "latency_ms": 35191.101, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "src/flask/sansio/app.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.1667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 8, + "tokens_delivered": 2036, + "latency_ms": 35155.593, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 512, + "latency_ms": 50613.616, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [ + "src/flask/sansio/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1045, + "latency_ms": 35560.112, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 897, + "latency_ms": 34859.779, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/RW-gin.json b/benchmarks/context-retrieval/results/RW-gin.json new file mode 100644 index 00000000..9091f30e --- /dev/null +++ b/benchmarks/context-retrieval/results/RW-gin.json @@ -0,0 +1,6630 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "tier": "small", + "tier_code_files": 99, + "corpus_counts": { + "cases": 76, + "path_leak": 23, + "baseline_blind": 53, + "multi_file": 54 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.4594, + "control": 0.0263, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "data race with trustedCIDRs (#2674) (#2675)", + "revision": "d496f64540b6707602de50ab57aeea8ff4080b74", + "returned": [ + "tree_test.go", + "binding/form_mapping_test.go", + "binding/form.go", + "routes_test.go", + "tree.go" + ] + }, + { + "provider_id": "random-files", + "query": "Fix #3500 Add escape logic for header (#3503)", + "revision": "fc1c43298de675e5252d0b44f97dc5e204bd4e1e", + "returned": [ + "mode_test.go", + "routes_test.go", + "internal/json/json.go", + "testdata/protoexample/test.pb.go", + "render/render.go" + ] + }, + { + "provider_id": "random-files", + "query": "check obj type in protobufBinding (#2851)", + "revision": "deb83b6365cf1b95f50929492f6058c94a6ff274", + "returned": [ + ".github/workflows/gin.yml", + "tree_test.go", + "render/xml.go", + "binding/protobuf.go", + "test_helpers.go" + ] + }, + { + "provider_id": "churn-ranked", + "query": "data race with trustedCIDRs (#2674) (#2675)", + "revision": "d496f64540b6707602de50ab57aeea8ff4080b74", + "returned": [ + "context.go", + "context_test.go", + "gin.go", + "binding/binding_test.go", + ".travis.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Do not panic when handling method not allowed on empty tree (#4003)", + "revision": "9c081de9cdd1948f521d47d170d18cbc2981c33a", + "returned": [ + "context.go", + "context_test.go", + "gin.go", + ".github/workflows/gin.yml", + "tree.go" + ] + }, + { + "provider_id": "churn-ranked", + "query": "check obj type in protobufBinding (#2851)", + "revision": "deb83b6365cf1b95f50929492f6058c94a6ff274", + "returned": [ + "context.go", + "context_test.go", + "gin.go", + "binding/binding_test.go", + "binding/form_mapping.go" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.068, + "full_recall_at_5": 0.0132, + "mean_recall_at_10": 0.0921, + "full_recall_at_10": 0.0263, + "mean_recall_at_20": 0.1776, + "full_recall_at_20": 0.0526, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0263, + "mean_recall_at_16000_tokens": 0.1009, + "mean_precision_at_10": 0.0197, + "zero_hit_rate": 0.6842, + "median_tokens_delivered": 24792.5, + "median_latency_ms": 12.089, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 74, + "mean_recall_at_5": 0.0563, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0676, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1554, + "full_recall_at_20": 0.027, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.027, + "mean_recall_at_16000_tokens": 0.0766, + "mean_precision_at_10": 0.0162, + "zero_hit_rate": 0.7027, + "median_tokens_delivered": 24792.5, + "median_latency_ms": 12.135, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.15, + "zero_hit_rate": 0, + "median_tokens_delivered": 21401, + "median_latency_ms": 10.7245, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.0435, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0652, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1522, + "full_recall_at_20": 0.0435, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0217, + "mean_recall_at_16000_tokens": 0.0652, + "mean_precision_at_10": 0.013, + "zero_hit_rate": 0.7391, + "median_tokens_delivered": 24180, + "median_latency_ms": 11.94, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.0786, + "full_recall_at_5": 0.0189, + "mean_recall_at_10": 0.1038, + "full_recall_at_10": 0.0377, + "mean_recall_at_20": 0.1887, + "full_recall_at_20": 0.0566, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0283, + "mean_recall_at_16000_tokens": 0.1164, + "mean_precision_at_10": 0.0226, + "zero_hit_rate": 0.6604, + "median_tokens_delivered": 25977, + "median_latency_ms": 12.091, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 76 + }, + "resources": { + "baseline_rss_mb": 21695.6, + "peak_rss_mb": 21814.2, + "delta_rss_mb": 118.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.3783, + "full_recall_at_5": 0.2632, + "mean_recall_at_10": 0.5077, + "full_recall_at_10": 0.3553, + "mean_recall_at_20": 0.7368, + "full_recall_at_20": 0.6579, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0175, + "mean_recall_at_16000_tokens": 0.2884, + "mean_precision_at_10": 0.0934, + "zero_hit_rate": 0.1842, + "median_tokens_delivered": 88454.5, + "median_latency_ms": 24.7655, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 74, + "mean_recall_at_5": 0.375, + "full_recall_at_5": 0.2568, + "mean_recall_at_10": 0.5079, + "full_recall_at_10": 0.3514, + "mean_recall_at_20": 0.7297, + "full_recall_at_20": 0.6486, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.018, + "mean_recall_at_16000_tokens": 0.2962, + "mean_precision_at_10": 0.0946, + "zero_hit_rate": 0.1892, + "median_tokens_delivered": 88454.5, + "median_latency_ms": 24.7655, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.05, + "zero_hit_rate": 0, + "median_tokens_delivered": 87302.5, + "median_latency_ms": 26.8095, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.4203, + "full_recall_at_5": 0.3478, + "mean_recall_at_10": 0.5507, + "full_recall_at_10": 0.3913, + "mean_recall_at_20": 0.7174, + "full_recall_at_20": 0.6087, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0435, + "mean_recall_at_16000_tokens": 0.3768, + "mean_precision_at_10": 0.0913, + "zero_hit_rate": 0.1739, + "median_tokens_delivered": 90059, + "median_latency_ms": 23.587, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3601, + "full_recall_at_5": 0.2264, + "mean_recall_at_10": 0.489, + "full_recall_at_10": 0.3396, + "mean_recall_at_20": 0.7453, + "full_recall_at_20": 0.6792, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0063, + "mean_recall_at_16000_tokens": 0.25, + "mean_precision_at_10": 0.0943, + "zero_hit_rate": 0.1887, + "median_tokens_delivered": 88304, + "median_latency_ms": 24.896, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 76 + }, + "resources": { + "baseline_rss_mb": 21821.4, + "peak_rss_mb": 22199.9, + "delta_rss_mb": 378.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "repowise", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.3564, + "full_recall_at_5": 0.1316, + "mean_recall_at_10": 0.4529, + "full_recall_at_10": 0.2237, + "mean_recall_at_20": 0.4594, + "full_recall_at_20": 0.2237, + "mean_recall_at_1000_tokens": 0.4002, + "mean_recall_at_4000_tokens": 0.4594, + "mean_recall_at_16000_tokens": 0.4594, + "mean_precision_at_10": 0.1092, + "zero_hit_rate": 0.2895, + "median_tokens_delivered": 1507.5, + "median_latency_ms": 36644.7525, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 74, + "mean_recall_at_5": 0.3592, + "full_recall_at_5": 0.1351, + "mean_recall_at_10": 0.4448, + "full_recall_at_10": 0.2162, + "mean_recall_at_20": 0.4516, + "full_recall_at_20": 0.2162, + "mean_recall_at_1000_tokens": 0.3908, + "mean_recall_at_4000_tokens": 0.4516, + "mean_recall_at_16000_tokens": 0.4516, + "mean_precision_at_10": 0.1095, + "zero_hit_rate": 0.2973, + "median_tokens_delivered": 1502, + "median_latency_ms": 36644.7525, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.25, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.75, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.75, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0.75, + "mean_recall_at_4000_tokens": 0.75, + "mean_recall_at_16000_tokens": 0.75, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 1583.5, + "median_latency_ms": 36091.6065, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.3551, + "full_recall_at_5": 0.1304, + "mean_recall_at_10": 0.413, + "full_recall_at_10": 0.1304, + "mean_recall_at_20": 0.4348, + "full_recall_at_20": 0.1304, + "mean_recall_at_1000_tokens": 0.3696, + "mean_recall_at_4000_tokens": 0.4348, + "mean_recall_at_16000_tokens": 0.4348, + "mean_precision_at_10": 0.099, + "zero_hit_rate": 0.2609, + "median_tokens_delivered": 1559, + "median_latency_ms": 36443.147, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3569, + "full_recall_at_5": 0.1321, + "mean_recall_at_10": 0.4701, + "full_recall_at_10": 0.2642, + "mean_recall_at_20": 0.4701, + "full_recall_at_20": 0.2642, + "mean_recall_at_1000_tokens": 0.4135, + "mean_recall_at_4000_tokens": 0.4701, + "mean_recall_at_16000_tokens": 0.4701, + "mean_precision_at_10": 0.1136, + "zero_hit_rate": 0.3019, + "median_tokens_delivered": 1500, + "median_latency_ms": 36815.559, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 74, + "indexed-but-returned-nothing": 2 + }, + "resources": { + "baseline_rss_mb": 22223.1, + "peak_rss_mb": 24507.8, + "delta_rss_mb": 2284.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go", + "logger_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 44600, + "latency_ms": 25.462, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12761, + "latency_ms": 13.576, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26285, + "latency_ms": 11.599, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24674, + "latency_ms": 12.631, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21162, + "latency_ms": 12.467, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 27850, + "latency_ms": 12.663, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28556, + "latency_ms": 12.389, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18035, + "latency_ms": 13.096, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19033, + "latency_ms": 13.478, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 37122, + "latency_ms": 12.204, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30316, + "latency_ms": 12.357, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 33394, + "latency_ms": 12.087, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26404, + "latency_ms": 12.218, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26292, + "latency_ms": 12.337, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22473, + "latency_ms": 12.327, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 15956, + "latency_ms": 12.457, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19388, + "latency_ms": 12.56, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17445, + "latency_ms": 12.482, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go" + ], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 27773, + "latency_ms": 13.084, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14497, + "latency_ms": 12.624, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30553, + "latency_ms": 13.265, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 37728, + "latency_ms": 15.487, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 34527, + "latency_ms": 11.561, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19547, + "latency_ms": 12.559, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12483, + "latency_ms": 13.174, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 22479, + "latency_ms": 13.033, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14254, + "latency_ms": 12.928, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 35496, + "latency_ms": 12.797, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [], + "missed": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23550, + "latency_ms": 12.955, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18818, + "latency_ms": 14.039, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 48958, + "latency_ms": 12.613, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53000, + "latency_ms": 11.838, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 28956, + "latency_ms": 13.041, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21545, + "latency_ms": 11.552, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 34999, + "latency_ms": 11.587, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 26477, + "latency_ms": 12.179, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20935, + "latency_ms": 11.047, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20327, + "latency_ms": 11.94, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21566, + "latency_ms": 10.725, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25538, + "latency_ms": 11.214, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24170, + "latency_ms": 10.444, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23086, + "latency_ms": 11.183, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20136, + "latency_ms": 10.813, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8753, + "latency_ms": 11.555, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35582, + "latency_ms": 13.715, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 27255, + "latency_ms": 12.383, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30122, + "latency_ms": 12.04, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [], + "missed": [ + "gin.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19879, + "latency_ms": 12.018, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33426, + "latency_ms": 11.523, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16642, + "latency_ms": 11.022, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27618, + "latency_ms": 10.537, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "errors.go", + "routergroup.go" + ], + "missed": [ + "context.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 39163, + "latency_ms": 12.359, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 16118, + "latency_ms": 12.974, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23279, + "latency_ms": 12.571, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 36634, + "latency_ms": 11.672, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 40307, + "latency_ms": 11.201, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23860, + "latency_ms": 11.183, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37903, + "latency_ms": 10.507, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49541, + "latency_ms": 11.403, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44167, + "latency_ms": 10.913, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13012, + "latency_ms": 11.512, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28129, + "latency_ms": 11.288, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19778, + "latency_ms": 10.93, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 27314, + "latency_ms": 11.105, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22773, + "latency_ms": 11.209, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 28023, + "latency_ms": 11.962, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24288, + "latency_ms": 11.73, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26606, + "latency_ms": 12.091, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21083, + "latency_ms": 11.954, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8705, + "latency_ms": 12.334, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55452, + "latency_ms": 11.928, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 25977, + "latency_ms": 11.01, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/protobuf.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 24180, + "latency_ms": 10.731, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16819, + "latency_ms": 10.797, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 15488, + "latency_ms": 10.344, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24911, + "latency_ms": 13.192, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 69810, + "latency_ms": 24.761, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 84576, + "latency_ms": 21.648, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87509, + "latency_ms": 22.161, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 82399, + "latency_ms": 22.408, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 83992, + "latency_ms": 22.355, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 72743, + "latency_ms": 24.896, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 88246, + "latency_ms": 24.078, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105412, + "latency_ms": 23.393, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 84449, + "latency_ms": 23.345, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 95317, + "latency_ms": 22.842, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 69979, + "latency_ms": 26.883, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 113276, + "latency_ms": 23.785, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 108509, + "latency_ms": 24.121, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 84446, + "latency_ms": 22.151, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 90304, + "latency_ms": 23.313, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 89875, + "latency_ms": 23.266, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70138, + "latency_ms": 24.573, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93950, + "latency_ms": 24.55, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 87314, + "latency_ms": 22.975, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82768, + "latency_ms": 23.618, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 88101, + "latency_ms": 22.588, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 87276, + "latency_ms": 25.467, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 117055, + "latency_ms": 23.408, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 113658, + "latency_ms": 23.61, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 87849, + "latency_ms": 22.313, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 89097, + "latency_ms": 22.692, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93972, + "latency_ms": 24.77, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 88010, + "latency_ms": 24.643, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "test_helpers.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 88138, + "latency_ms": 23.587, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 103341, + "latency_ms": 22.124, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 115951, + "latency_ms": 25.57, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 106775, + "latency_ms": 24.705, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 94928, + "latency_ms": 25.96, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 110221, + "latency_ms": 25.926, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87531, + "latency_ms": 23.392, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 102446, + "latency_ms": 24.081, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 83359, + "latency_ms": 23.278, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 66540, + "latency_ms": 23.576, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 101365, + "latency_ms": 24.695, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 88522, + "latency_ms": 23.569, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 92380, + "latency_ms": 23.124, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 82724, + "latency_ms": 22.886, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 88010, + "latency_ms": 25.064, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 84577, + "latency_ms": 24.848, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 88304, + "latency_ms": 27.525, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 88116, + "latency_ms": 27.901, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 98912, + "latency_ms": 28.516, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 86657, + "latency_ms": 27.612, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 90329, + "latency_ms": 28.031, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88014, + "latency_ms": 30.431, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 112651, + "latency_ms": 29.342, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "recovery_test.go" + ], + "missed": [ + "errors.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 115859, + "latency_ms": 29.283, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88387, + "latency_ms": 29.155, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 82791, + "latency_ms": 36.021, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 111258, + "latency_ms": 29.81, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88075, + "latency_ms": 30.909, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86659, + "latency_ms": 31.25, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 88259, + "latency_ms": 30.965, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 101364, + "latency_ms": 29.276, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 108196, + "latency_ms": 31.173, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 90059, + "latency_ms": 30.326, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [ + "recovery.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 72362, + "latency_ms": 29.637, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 105612, + "latency_ms": 35.671, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 106371, + "latency_ms": 29.89, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 92820, + "latency_ms": 29.198, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 83426, + "latency_ms": 29.713, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 84119, + "latency_ms": 29.064, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 113276, + "latency_ms": 27.798, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 113946, + "latency_ms": 28.139, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93998, + "latency_ms": 27.914, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 98912, + "latency_ms": 26.505, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 98262, + "latency_ms": 25.522, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "binding/protobuf.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 82704, + "latency_ms": 26.021, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 99455, + "latency_ms": 24.429, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 68234, + "latency_ms": 23.729, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 90299, + "latency_ms": 22.142, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go", + "logger_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 1618, + "latency_ms": 36994.582, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "debug_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1494, + "latency_ms": 37824.299, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2249, + "latency_ms": 37454.273, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 339, + "latency_ms": 37027.346, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 314, + "latency_ms": 37425.17, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1529, + "latency_ms": 37380.335, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5, + "latency_ms": 38561.738, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go" + ], + "missed": [ + "response_writer_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1928, + "latency_ms": 39257.545, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 844, + "latency_ms": 36996.877, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1354, + "latency_ms": 37365.068, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1630, + "latency_ms": 37123.436, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2132, + "latency_ms": 38830.004, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1462, + "latency_ms": 37376.503, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1026, + "latency_ms": 36198.754, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2064, + "latency_ms": 36601.141, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1329, + "latency_ms": 36575.898, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 332, + "latency_ms": 36620.998, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1523, + "latency_ms": 37667.181, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.25, + "recall_at_20": 0.6667, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 1619, + "latency_ms": 37366.669, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2514, + "latency_ms": 37888.117, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1778, + "latency_ms": 38261.79, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1797, + "latency_ms": 37105.393, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 1249, + "latency_ms": 37998.597, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1209, + "latency_ms": 38456.107, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2206, + "latency_ms": 36568.548, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2336, + "latency_ms": 37716.429, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2372, + "latency_ms": 37075.243, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "response_writer.go" + ], + "missed": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.1667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 7, + "tokens_delivered": 2066, + "latency_ms": 36166.71, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "test_helpers.go" + ], + "missed": [ + "context_test.go", + "gin.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 7, + "tokens_delivered": 1256, + "latency_ms": 36090.507, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 1842, + "latency_ms": 36581.573, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go" + ], + "missed": [ + "render/render_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1738, + "latency_ms": 36727.807, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1489, + "latency_ms": 36520.901, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1500, + "latency_ms": 36071.311, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1466, + "latency_ms": 36380.692, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2135, + "latency_ms": 35598.386, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1511, + "latency_ms": 38317.183, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1559, + "latency_ms": 50531.057, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5, + "latency_ms": 35082.146, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2222, + "recall_at_20": 0.5, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1337, + "latency_ms": 36443.147, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2048, + "latency_ms": 36145.273, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go" + ], + "missed": [ + "mode_test.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 1813, + "latency_ms": 36366.479, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1717, + "latency_ms": 37152.213, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1504, + "latency_ms": 36668.507, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 529, + "latency_ms": 36330.712, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go" + ], + "missed": [ + "binding/validate_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1273, + "latency_ms": 36252.317, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1417, + "latency_ms": 36564.743, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2042, + "latency_ms": 37032.367, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1434, + "latency_ms": 36365.411, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1622, + "latency_ms": 36016.156, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 457, + "latency_ms": 36361.896, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1654, + "latency_ms": 38855.693, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1429, + "recall_at_20": 0.25, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 2006, + "latency_ms": 37434.906, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 1788, + "latency_ms": 36289.257, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 327, + "latency_ms": 36112.541, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1256, + "latency_ms": 37294.587, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 410, + "latency_ms": 36052.347, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1721, + "latency_ms": 36272.876, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1266, + "latency_ms": 36333.567, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go" + ], + "missed": [ + "errors_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1994, + "latency_ms": 36815.559, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 1281, + "latency_ms": 39067.185, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1415, + "latency_ms": 36596.56, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2403, + "latency_ms": 35657.616, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 1967, + "latency_ms": 37009.579, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1528, + "latency_ms": 36728.218, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2375, + "latency_ms": 36309.432, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 913, + "latency_ms": 36127.19, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 878, + "latency_ms": 35905.329, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1316, + "latency_ms": 37056.981, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1190, + "latency_ms": 36964.962, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1460, + "latency_ms": 36067.375, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1945, + "latency_ms": 36224.326, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1687, + "latency_ms": 36923.964, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/protobuf.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1072, + "latency_ms": 35287.661, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 670, + "latency_ms": 36041.848, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "debug_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1639, + "latency_ms": 35454.995, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 712, + "latency_ms": 35743.632, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/RW-got.json b/benchmarks/context-retrieval/results/RW-got.json new file mode 100644 index 00000000..d95669bc --- /dev/null +++ b/benchmarks/context-retrieval/results/RW-got.json @@ -0,0 +1,9373 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.4582, + "control": 0.0296, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "source/as-promise/core.ts", + "source/as-promise/calculate-retry-delay.ts", + "source/core/utils/timed-out.ts", + "test/cookies.ts", + "test/types/create-test-server/index.d.ts" + ] + }, + { + "provider_id": "random-files", + "query": "Fix hook type definitions to reflect normalized runtime state", + "revision": "60a441939af36143d1ead87b79ba706fe0a5b812", + "returned": [ + "source/core/utils/is-form-data.ts", + "documentation/examples/runkit-example.js", + "test/helpers/with-server.ts", + "test/helpers/create-https-test-server.ts", + "source/core/parse-link-header.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "package.json", + "source/core/index.ts", + "source/create.ts", + "test/timeout.ts", + "test/arguments.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Handle abort signals added by handlers", + "revision": "f6a058a7d1fdd0b65bb75db9faf94490fb7a66ec", + "returned": [ + "source/core/index.ts", + "package.json", + "source/core/options.ts", + "test/hooks.ts", + "test/stream.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix TypeScript type inference for `got.extend()` with `responseType` option", + "revision": "a149dfbf72fedc3822cc9f8157582f6b92d43774", + "returned": [ + "source/core/index.ts", + "package.json", + "source/core/options.ts", + "test/stream.ts", + "test/hooks.ts" + ] + }, + { + "provider_id": "repowise", + "query": "Fix encoding with json `responseType` (#1996)", + "revision": "dbbd31777b3299ceb0db99798453e01ab8e597b1", + "returned": [ + "source/core/options.ts", + "source/core/response.ts", + "source/as-promise/index.ts", + "source/types.ts", + "documentation/examples/advanced-creation.js" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.0968, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.1927, + "full_recall_at_10": 0.0741, + "mean_recall_at_20": 0.359, + "full_recall_at_20": 0.1481, + "mean_recall_at_1000_tokens": 0.0031, + "mean_recall_at_4000_tokens": 0.0296, + "mean_recall_at_16000_tokens": 0.1954, + "mean_precision_at_10": 0.0361, + "zero_hit_rate": 0.4167, + "median_tokens_delivered": 29954, + "median_latency_ms": 11.9375, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 100, + "mean_recall_at_5": 0.0645, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1282, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3077, + "full_recall_at_20": 0.08, + "mean_recall_at_1000_tokens": 0.0033, + "mean_recall_at_4000_tokens": 0.012, + "mean_recall_at_16000_tokens": 0.131, + "mean_precision_at_10": 0.031, + "zero_hit_rate": 0.45, + "median_tokens_delivered": 29656.5, + "median_latency_ms": 11.9375, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 8, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.25, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 34611, + "median_latency_ms": 11.9175, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.1302, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.242, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.3753, + "full_recall_at_20": 0.1667, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.016, + "mean_recall_at_16000_tokens": 0.2519, + "mean_precision_at_10": 0.0426, + "zero_hit_rate": 0.4074, + "median_tokens_delivered": 29934, + "median_latency_ms": 11.917, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.0633, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.1435, + "full_recall_at_10": 0.037, + "mean_recall_at_20": 0.3426, + "full_recall_at_20": 0.1296, + "mean_recall_at_1000_tokens": 0.0062, + "mean_recall_at_4000_tokens": 0.0432, + "mean_recall_at_16000_tokens": 0.1389, + "mean_precision_at_10": 0.0296, + "zero_hit_rate": 0.4259, + "median_tokens_delivered": 29954, + "median_latency_ms": 11.95, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 21696.4, + "peak_rss_mb": 21847.7, + "delta_rss_mb": 151.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.4293, + "full_recall_at_5": 0.1852, + "mean_recall_at_10": 0.5823, + "full_recall_at_10": 0.3426, + "mean_recall_at_20": 0.7793, + "full_recall_at_20": 0.6019, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0278, + "mean_recall_at_16000_tokens": 0.3735, + "mean_precision_at_10": 0.1157, + "zero_hit_rate": 0.0741, + "median_tokens_delivered": 76435, + "median_latency_ms": 28.7385, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 100, + "mean_recall_at_5": 0.4437, + "full_recall_at_5": 0.18, + "mean_recall_at_10": 0.5888, + "full_recall_at_10": 0.33, + "mean_recall_at_20": 0.7817, + "full_recall_at_20": 0.59, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.01, + "mean_recall_at_16000_tokens": 0.3833, + "mean_precision_at_10": 0.121, + "zero_hit_rate": 0.06, + "median_tokens_delivered": 76435, + "median_latency_ms": 28.7025, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 8, + "mean_recall_at_5": 0.25, + "full_recall_at_5": 0.25, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.75, + "full_recall_at_20": 0.75, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.25, + "mean_recall_at_16000_tokens": 0.25, + "mean_precision_at_10": 0.05, + "zero_hit_rate": 0.25, + "median_tokens_delivered": 75185.5, + "median_latency_ms": 31.1875, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3401, + "full_recall_at_5": 0.0926, + "mean_recall_at_10": 0.4963, + "full_recall_at_10": 0.2037, + "mean_recall_at_20": 0.7161, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0093, + "mean_recall_at_16000_tokens": 0.2778, + "mean_precision_at_10": 0.1056, + "zero_hit_rate": 0.0926, + "median_tokens_delivered": 76984, + "median_latency_ms": 28.762, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.5185, + "full_recall_at_5": 0.2778, + "mean_recall_at_10": 0.6682, + "full_recall_at_10": 0.4815, + "mean_recall_at_20": 0.8426, + "full_recall_at_20": 0.7037, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0463, + "mean_recall_at_16000_tokens": 0.4691, + "mean_precision_at_10": 0.1259, + "zero_hit_rate": 0.0556, + "median_tokens_delivered": 75083.5, + "median_latency_ms": 28.719, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 21884.6, + "peak_rss_mb": 23358.4, + "delta_rss_mb": 1473.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "repowise", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.3491, + "full_recall_at_5": 0.1296, + "mean_recall_at_10": 0.4319, + "full_recall_at_10": 0.1667, + "mean_recall_at_20": 0.4582, + "full_recall_at_20": 0.1944, + "mean_recall_at_1000_tokens": 0.3414, + "mean_recall_at_4000_tokens": 0.4582, + "mean_recall_at_16000_tokens": 0.4582, + "mean_precision_at_10": 0.0905, + "zero_hit_rate": 0.25, + "median_tokens_delivered": 1783, + "median_latency_ms": 34934.3795, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 100, + "mean_recall_at_5": 0.337, + "full_recall_at_5": 0.1, + "mean_recall_at_10": 0.4265, + "full_recall_at_10": 0.14, + "mean_recall_at_20": 0.4548, + "full_recall_at_20": 0.17, + "mean_recall_at_1000_tokens": 0.3287, + "mean_recall_at_4000_tokens": 0.4548, + "mean_recall_at_16000_tokens": 0.4548, + "mean_precision_at_10": 0.0913, + "zero_hit_rate": 0.23, + "median_tokens_delivered": 1772.5, + "median_latency_ms": 35067.4915, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 8, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.5, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0.5, + "mean_recall_at_4000_tokens": 0.5, + "mean_recall_at_16000_tokens": 0.5, + "mean_precision_at_10": 0.0806, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 2002, + "median_latency_ms": 34693.0835, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3401, + "full_recall_at_5": 0.0926, + "mean_recall_at_10": 0.4302, + "full_recall_at_10": 0.1481, + "mean_recall_at_20": 0.4642, + "full_recall_at_20": 0.1852, + "mean_recall_at_1000_tokens": 0.3309, + "mean_recall_at_4000_tokens": 0.4642, + "mean_recall_at_16000_tokens": 0.4642, + "mean_precision_at_10": 0.0875, + "zero_hit_rate": 0.2407, + "median_tokens_delivered": 1888, + "median_latency_ms": 34847.358, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.358, + "full_recall_at_5": 0.1667, + "mean_recall_at_10": 0.4336, + "full_recall_at_10": 0.1852, + "mean_recall_at_20": 0.4522, + "full_recall_at_20": 0.2037, + "mean_recall_at_1000_tokens": 0.3519, + "mean_recall_at_4000_tokens": 0.4522, + "mean_recall_at_16000_tokens": 0.4522, + "mean_precision_at_10": 0.0936, + "zero_hit_rate": 0.2593, + "median_tokens_delivered": 1651, + "median_latency_ms": 35173.3525, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 107, + "indexed-but-returned-nothing": 1 + }, + "resources": { + "baseline_rss_mb": 23023.3, + "peak_rss_mb": 24498.1, + "delta_rss_mb": 1474.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "test/create.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 17695, + "latency_ms": 25.557, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29330, + "latency_ms": 13.611, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44537, + "latency_ms": 11.46, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 30339, + "latency_ms": 12.901, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28268, + "latency_ms": 12.445, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 37092, + "latency_ms": 12.616, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24761, + "latency_ms": 12.333, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 32998, + "latency_ms": 13.508, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 45500, + "latency_ms": 13.013, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 40700, + "latency_ms": 12.341, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 63090, + "latency_ms": 12.354, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29834, + "latency_ms": 11.905, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 19371, + "latency_ms": 12.279, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26240, + "latency_ms": 12.331, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 41881, + "latency_ms": 12.161, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40853, + "latency_ms": 12.572, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 41220, + "latency_ms": 12.617, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 19500, + "latency_ms": 12.478, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 40496, + "latency_ms": 12.964, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18203, + "latency_ms": 12.793, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24438, + "latency_ms": 13.416, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20036, + "latency_ms": 14.588, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20481, + "latency_ms": 12.282, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 27897, + "latency_ms": 12.607, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 48003, + "latency_ms": 13.127, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "tsconfig.json" + ], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 44800, + "latency_ms": 13.055, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25732, + "latency_ms": 13.099, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19504, + "latency_ms": 12.638, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29947, + "latency_ms": 13.61, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17080, + "latency_ms": 13.382, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 42253, + "latency_ms": 12.531, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 50293, + "latency_ms": 11.944, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 17, + "tokens_delivered": 16939, + "latency_ms": 12.965, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 64594, + "latency_ms": 11.598, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [ + "source/as-promise/types.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 20438, + "latency_ms": 11.688, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19771, + "latency_ms": 12.007, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32900, + "latency_ms": 11.106, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 36724, + "latency_ms": 11.53, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [ + "test/agent.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 29961, + "latency_ms": 11.125, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29680, + "latency_ms": 11.194, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 45651, + "latency_ms": 10.415, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 20651, + "latency_ms": 11.202, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 26908, + "latency_ms": 10.98, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14053, + "latency_ms": 11.637, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts" + ], + "missed": [ + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 23093, + "latency_ms": 13.232, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 34438, + "latency_ms": 12.45, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 40983, + "latency_ms": 12.086, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 7, + "tokens_delivered": 38400, + "latency_ms": 11.956, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 56327, + "latency_ms": 11.585, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/error.ts", + "test/helpers.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 2, + "tokens_delivered": 27556, + "latency_ms": 11.073, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31766, + "latency_ms": 10.704, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 9, + "tokens_delivered": 25946, + "latency_ms": 12.376, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 61131, + "latency_ms": 12.903, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43051, + "latency_ms": 12.458, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 55773, + "latency_ms": 11.775, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43907, + "latency_ms": 11.186, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 26684, + "latency_ms": 11.096, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 28696, + "latency_ms": 10.751, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 29633, + "latency_ms": 11.304, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31568, + "latency_ms": 10.873, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 32483, + "latency_ms": 11.542, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 16998, + "latency_ms": 11.311, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 28802, + "latency_ms": 10.993, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18220, + "latency_ms": 11.002, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 34057, + "latency_ms": 11.19, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27444, + "latency_ms": 11.978, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [ + "test/progress.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 43111, + "latency_ms": 11.889, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 36388, + "latency_ms": 11.946, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 26052, + "latency_ms": 11.933, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34828, + "latency_ms": 12.348, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 47799, + "latency_ms": 11.929, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15060, + "latency_ms": 11.042, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35728, + "latency_ms": 10.714, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30113, + "latency_ms": 10.487, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 26343, + "latency_ms": 10.531, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [ + "test/early-307-redirects.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 66990, + "latency_ms": 13.179, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 33984, + "latency_ms": 12.521, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 70809, + "latency_ms": 12.408, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16148, + "latency_ms": 12.016, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22493, + "latency_ms": 11.538, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43364, + "latency_ms": 10.958, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25107, + "latency_ms": 9.987, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 34784, + "latency_ms": 10.331, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21815, + "latency_ms": 10.804, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 39157, + "latency_ms": 10.551, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21610, + "latency_ms": 11.098, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 36075, + "latency_ms": 10.677, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24832, + "latency_ms": 10.82, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/options.ts", + "test/headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 32152, + "latency_ms": 11.135, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 25999, + "latency_ms": 11.942, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16382, + "latency_ms": 12.556, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37626, + "latency_ms": 12.494, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 57177, + "latency_ms": 11.269, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 34640, + "latency_ms": 10.653, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24940, + "latency_ms": 13.03, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 39937, + "latency_ms": 11.674, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 42625, + "latency_ms": 11.564, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 30034, + "latency_ms": 11.135, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "source/core/index.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 26779, + "latency_ms": 11.712, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "test/retry.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 40775, + "latency_ms": 13.77, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13037, + "latency_ms": 13.073, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23583, + "latency_ms": 11.627, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27860, + "latency_ms": 12.19, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24606, + "latency_ms": 12.158, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28939, + "latency_ms": 11.884, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "test/extend.types.ts" + ], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 11, + "tokens_delivered": 24896, + "latency_ms": 11.394, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 56437, + "latency_ms": 10.746, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17646, + "latency_ms": 11.499, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 58679, + "latency_ms": 29.973, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 89852, + "latency_ms": 24.666, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 82176, + "latency_ms": 22.812, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 57240, + "latency_ms": 25.891, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73871, + "latency_ms": 22.999, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/gzip.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 162539, + "latency_ms": 26.603, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 87852, + "latency_ms": 23.007, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 73650, + "latency_ms": 23.801, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 103569, + "latency_ms": 22.716, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 71989, + "latency_ms": 25.412, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 162719, + "latency_ms": 24.503, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55161, + "latency_ms": 26.732, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 49756, + "latency_ms": 25.157, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 69499, + "latency_ms": 25.59, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 57290, + "latency_ms": 27.026, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 93628, + "latency_ms": 26.316, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 57286, + "latency_ms": 28.145, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71767, + "latency_ms": 24.221, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 91285, + "latency_ms": 24.63, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 77785, + "latency_ms": 23.8, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 88524, + "latency_ms": 23.732, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 53380, + "latency_ms": 25.014, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 91140, + "latency_ms": 24.271, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 76834, + "latency_ms": 24.543, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 101370, + "latency_ms": 24.33, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 76435, + "latency_ms": 26.439, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 76435, + "latency_ms": 24.86, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 87924, + "latency_ms": 27.509, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 76166, + "latency_ms": 28.683, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 68859, + "latency_ms": 31.499, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 71393, + "latency_ms": 30.334, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 70103, + "latency_ms": 28.977, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 51379, + "latency_ms": 31.342, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 103622, + "latency_ms": 28.755, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70281, + "latency_ms": 31.735, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [ + "source/types.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 103976, + "latency_ms": 29.988, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 87647, + "latency_ms": 33.291, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54612, + "latency_ms": 32.893, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 50065, + "latency_ms": 33.448, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 163314, + "latency_ms": 32.192, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 72182, + "latency_ms": 30.85, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 89284, + "latency_ms": 31.161, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 103622, + "latency_ms": 30.124, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 57236, + "latency_ms": 31.714, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 56354, + "latency_ms": 30.663, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 77422, + "latency_ms": 32.691, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 84288, + "latency_ms": 30.425, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 89510, + "latency_ms": 28.883, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 90189, + "latency_ms": 29.197, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 57196, + "latency_ms": 33.013, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 82777, + "latency_ms": 29.396, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts" + ], + "missed": [ + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 55092, + "latency_ms": 30.042, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 162744, + "latency_ms": 26.351, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 55701, + "latency_ms": 27.805, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 82287, + "latency_ms": 27.555, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/utils/get-body-size.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 69971, + "latency_ms": 26.864, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 49545, + "latency_ms": 25.555, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 54988, + "latency_ms": 24.75, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 51355, + "latency_ms": 27.025, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 69534, + "latency_ms": 28.679, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 55421, + "latency_ms": 26.658, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 55679, + "latency_ms": 26.101, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 51051, + "latency_ms": 26.002, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82219, + "latency_ms": 25.187, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 54539, + "latency_ms": 26.212, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 94625, + "latency_ms": 27.73, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73537, + "latency_ms": 24.797, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 71935, + "latency_ms": 23.916, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 58746, + "latency_ms": 26.201, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82168, + "latency_ms": 26.116, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 57062, + "latency_ms": 25.811, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 87789, + "latency_ms": 26.334, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "test/post.ts" + ], + "missed": [ + "source/core/utils/get-body-size.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 88149, + "latency_ms": 38.109, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 54889, + "latency_ms": 46.17, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 56081, + "latency_ms": 48.76, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [ + "test/early-307-redirects.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 163538, + "latency_ms": 45.114, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 77420, + "latency_ms": 46.474, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 161120, + "latency_ms": 42.572, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 77692, + "latency_ms": 37.678, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 73848, + "latency_ms": 40.936, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 95064, + "latency_ms": 41.987, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 82287, + "latency_ms": 41.848, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 77134, + "latency_ms": 33.683, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 79062, + "latency_ms": 28.802, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 69515, + "latency_ms": 28.93, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 72804, + "latency_ms": 25.219, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82400, + "latency_ms": 40.578, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 81765, + "latency_ms": 38.132, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 72116, + "latency_ms": 32.269, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 53420, + "latency_ms": 42.037, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 55889, + "latency_ms": 32.206, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 82176, + "latency_ms": 28.722, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82513, + "latency_ms": 27.519, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 82131, + "latency_ms": 27.901, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 57497, + "latency_ms": 30.872, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 90031, + "latency_ms": 28.483, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 72711, + "latency_ms": 29.324, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 56960, + "latency_ms": 31.525, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 93102, + "latency_ms": 28.488, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 86056, + "latency_ms": 30.152, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 74001, + "latency_ms": 34.937, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 73071, + "latency_ms": 37.192, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 97394, + "latency_ms": 35.408, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82168, + "latency_ms": 36.372, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "source/create.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 82112, + "latency_ms": 40.024, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "test/create.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "missed": [ + "source/types.ts", + "test/extend.types.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 103627, + "latency_ms": 36.998, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 87367, + "latency_ms": 33.507, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 70574, + "latency_ms": 33.62, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 904, + "latency_ms": 34930.091, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2460, + "latency_ms": 37362.976, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1435, + "latency_ms": 36799.548, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1736, + "latency_ms": 36694.205, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1640, + "latency_ms": 37237.585, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 1779, + "latency_ms": 38756.988, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2229, + "latency_ms": 37785.284, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1513, + "latency_ms": 37821.705, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1516, + "latency_ms": 37848.928, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1157, + "latency_ms": 37160.691, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2054, + "latency_ms": 38565.687, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1668, + "latency_ms": 36634.299, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 1481, + "latency_ms": 36430.313, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1940, + "latency_ms": 36444.016, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2185, + "latency_ms": 36091.511, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1723, + "latency_ms": 36717.425, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 1464, + "latency_ms": 36637.251, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2574, + "latency_ms": 36804.584, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1382, + "latency_ms": 37343.661, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2308, + "latency_ms": 36838.941, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1150, + "latency_ms": 38072.889, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1360, + "latency_ms": 36470.581, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1314, + "latency_ms": 36594.378, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2086, + "latency_ms": 36001.946, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1932, + "latency_ms": 36443.613, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 420, + "latency_ms": 35903.371, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2021, + "latency_ms": 35108.931, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2529, + "latency_ms": 34879.845, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 1800, + "latency_ms": 36439.143, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 2093, + "latency_ms": 36181.272, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "source/core/response.ts" + ], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 785, + "latency_ms": 35243.86, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1670, + "latency_ms": 34022.28, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 2693, + "latency_ms": 33665.92, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 2488, + "latency_ms": 34759.91, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2439, + "latency_ms": 34045.36, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 1559, + "latency_ms": 34784.801, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1247, + "latency_ms": 37271.06, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2128, + "latency_ms": 48388.713, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1385, + "latency_ms": 33617.466, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 1681, + "latency_ms": 35855.529, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1905, + "latency_ms": 34444.741, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1859, + "latency_ms": 34835.843, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1662, + "latency_ms": 36353.367, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1655, + "latency_ms": 34889.605, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2494, + "latency_ms": 35550.484, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1345, + "latency_ms": 35102.845, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2019, + "latency_ms": 35179.99, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 2324, + "latency_ms": 35446.317, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 5, + "tokens_delivered": 1851, + "latency_ms": 34906.779, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "source/core/errors.ts" + ], + "missed": [ + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 1806, + "latency_ms": 34695.262, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2054, + "latency_ms": 34931.658, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/promise.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1780, + "latency_ms": 34310.859, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 1868, + "latency_ms": 36503.23, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1951, + "latency_ms": 34599.013, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts" + ], + "missed": [ + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 1824, + "latency_ms": 34819.824, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 882, + "latency_ms": 34363.854, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1819, + "latency_ms": 34215.029, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2469, + "latency_ms": 34817.193, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 518, + "latency_ms": 34233.807, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 7, + "tokens_delivered": 1956, + "latency_ms": 34463.744, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 2128, + "latency_ms": 34120.862, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 1636, + "latency_ms": 35026.052, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1786, + "latency_ms": 35273.247, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1922, + "latency_ms": 34804.288, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1731, + "latency_ms": 34830.113, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 1482, + "latency_ms": 34769.757, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2330, + "latency_ms": 34864.603, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2161, + "latency_ms": 34521.564, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 1871, + "latency_ms": 34605.904, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1647, + "latency_ms": 34650.517, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1639, + "latency_ms": 34205.063, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1535, + "latency_ms": 34514.817, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1642, + "latency_ms": 34613.184, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 1906, + "latency_ms": 34029.747, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1527, + "latency_ms": 33786.388, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 9, + "tokens_delivered": 1949, + "latency_ms": 35750.869, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 482, + "latency_ms": 34435.675, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2104, + "latency_ms": 35886.79, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1674, + "latency_ms": 34373.014, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1292, + "latency_ms": 34428.074, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1908, + "latency_ms": 34673.767, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1865, + "latency_ms": 34504.901, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2326, + "latency_ms": 34937.101, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1163, + "latency_ms": 34433.305, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 1544, + "latency_ms": 34116.778, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1266, + "latency_ms": 34192.746, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1595, + "latency_ms": 34400.907, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2222, + "latency_ms": 34508.759, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 1637, + "latency_ms": 34248.689, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1885, + "latency_ms": 34461.406, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2125, + "latency_ms": 35129.582, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1585, + "latency_ms": 35806.777, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1444, + "latency_ms": 34379.342, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 748, + "latency_ms": 35888.519, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1733, + "latency_ms": 33954.558, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 2035, + "latency_ms": 36527.148, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2362, + "latency_ms": 35866.989, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1918, + "latency_ms": 33999.744, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 1743, + "latency_ms": 34720.156, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2276, + "latency_ms": 36024.84, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1845, + "latency_ms": 35853.888, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 969, + "latency_ms": 34358.726, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1489, + "latency_ms": 36134.183, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2521, + "latency_ms": 34758.965, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 1953, + "latency_ms": 34388.318, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts" + ], + "missed": [ + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 6, + "tokens_delivered": 2372, + "latency_ms": 36581.881, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1357, + "latency_ms": 36845.653, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1766, + "latency_ms": 36859.654, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/T2-fastify.json b/benchmarks/context-retrieval/results/T2-fastify.json new file mode 100644 index 00000000..7741137e --- /dev/null +++ b/benchmarks/context-retrieval/results/T2-fastify.json @@ -0,0 +1,17017 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "fastify", + "head": "83e69762aff047940b376fc2a266d143499ef727" + }, + "corpus_counts": { + "cases": 82, + "path_leak": 47, + "baseline_blind": 35, + "multi_file": 66 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.3539, + "control": 0, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "replace AssertionError with FST_ERR_PLUGIN_DEPENDENCY_NOT_REGISTERED in checkDependencies (#6774)", + "revision": "27f4d6222dd945238e96c1fdaa632d5dc3c37b2c", + "returned": [ + ".github/workflows/coverage-nix.yml", + "examples/route-prefix.js", + "test/internals/hooks.test.js", + "lib/content-type.js", + "test/router-options.test.js" + ] + }, + { + "provider_id": "churn-ranked", + "query": "add http method override warning (#6879)", + "revision": "b93d611c7cefc2cd4a0b6f05cc864f2b6c397a6d", + "returned": [ + "package.json", + "fastify.js", + ".github/workflows/ci.yml", + "lib/reply.js", + ".github/workflows/links-check.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "use JSON.stringify in onBadUrl for proper escaping (#6420)", + "revision": "970c575832521fff01cc018d928d84454811173b", + "returned": [ + "package.json", + "fastify.js", + ".github/workflows/ci.yml", + "lib/reply.js", + "lib/route.js" + ] + }, + { + "provider_id": "churn-ranked", + "query": "run preClose hook exactly once per declaring instance (#6940)", + "revision": "9a1535bcfe2a780cc6f9c6555df86e5072ca9a29", + "returned": [ + "package.json", + "fastify.js", + ".github/workflows/ci.yml", + "lib/reply.js", + "fastify.d.ts" + ] + }, + { + "provider_id": "keyword-search", + "query": "add http method override warning (#6879)", + "revision": "b93d611c7cefc2cd4a0b6f05cc864f2b6c397a6d", + "returned": [ + "fastify.js", + "test/content-parser.test.js", + "test/custom-parser.0.test.js", + "test/hooks.test.js", + "fastify.d.ts" + ] + }, + { + "provider_id": "keyword-search", + "query": "clear `[kState].readyPromise` for garbage collection (#6030)", + "revision": "7868e47194bcfb34cab2e51ea36bf10f764b538e", + "returned": [ + "fastify.js", + "lib/server.js", + "test/hooks.test.js", + "test/plugin.4.test.js", + "types/hooks.d.ts" + ] + }, + { + "provider_id": "keyword-search", + "query": "Align routerOptions defaultRoute types with runtime (#6392)", + "revision": "4c10980bd4a5a4d19e051f52a018779e6afb588b", + "returned": [ + "fastify.d.ts", + "fastify.js", + "build/build-validation.js", + "lib/route.js", + "types/instance.d.ts" + ] + }, + { + "provider_id": "agent-default", + "query": "add http method override warning (#6879)", + "revision": "b93d611c7cefc2cd4a0b6f05cc864f2b6c397a6d", + "returned": [ + "fastify.js", + "test/http2/unknown-http-method.test.js", + "lib/plugin-override.js", + "test/content-parser.test.js", + "test/custom-http-server.test.js" + ] + }, + { + "provider_id": "agent-default", + "query": "(types) allow FastifySchemaValidationError[] as an error (#6326)", + "revision": "9a67d3a4875e2c078f9f710035a7625819853677", + "returned": [ + "fastify.d.ts", + "fastify.js", + "build/build-validation.js", + "test/types/fastify.test-d.ts", + "test/schema-validation.test.js" + ] + }, + { + "provider_id": "graphify", + "query": "add http method override warning (#6879)", + "revision": "b93d611c7cefc2cd4a0b6f05cc864f2b6c397a6d", + "returned": [ + "fastify.js", + "lib/noop-set.js", + "docs/Reference/Server.md", + "test/custom-http-server.test.js", + "test/http2/unknown-http-method.test.js" + ] + }, + { + "provider_id": "graphify", + "query": "use ContentType to detect `json` and `charset` in reply.send (#6830)", + "revision": "75d74e1af74598deb18f2c052d64f815a5eec8f8", + "returned": [ + "lib/reply.js", + "package.json", + "lib/content-type.js", + "docs/Guides/Detecting-When-Clients-Abort.md", + "docs/Reference/Reply.md" + ] + }, + { + "provider_id": "token-savior-regex", + "query": "remove format placeholder from FST_ERR_CTP_INVALID_MEDIA_TYPE message (#6528)", + "revision": "8fad07272bf8042db8c0074c6fd40ed50fb6b0f7", + "returned": [ + ".github/scripts/lint-ecosystem.js", + "examples/benchmark/hooks-benchmark-async-await.js", + "examples/benchmark/hooks-benchmark.js", + "lib/content-type-parser.js", + "lib/content-type.js" + ] + }, + { + "provider_id": "token-savior-regex", + "query": "enable diagnostics tracking for async error handlers (#6458)", + "revision": "10c69d153c6ee4a61878eabfb2a8f3ec34ab9c8c", + "returned": [ + ".github/scripts/lint-ecosystem.js", + "examples/asyncawait.js", + "fastify.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ] + }, + { + "provider_id": "token-savior-regex", + "query": "error.code not present on some routing errors (#6674) (#6678)", + "revision": "c2ba18c17769dacce61472ec0397560b3b4eebde", + "returned": [ + ".github/scripts/lint-ecosystem.js", + "fastify.js", + "lib/route.js" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0024, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0329, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0091, + "mean_precision_at_10": 0.0012, + "zero_hit_rate": 0.9146, + "median_tokens_delivered": 30284.5, + "median_latency_ms": 11.725, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0024, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0329, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0091, + "mean_precision_at_10": 0.0012, + "zero_hit_rate": 0.9146, + "median_tokens_delivered": 30284.5, + "median_latency_ms": 11.725, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0043, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0255, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0053, + "mean_precision_at_10": 0.0021, + "zero_hit_rate": 0.9149, + "median_tokens_delivered": 32313, + "median_latency_ms": 11.736, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0429, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0143, + "mean_precision_at_10": 0, + "zero_hit_rate": 0.9143, + "median_tokens_delivered": 29377, + "median_latency_ms": 11.689, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 49966.7, + "peak_rss_mb": 50089.9, + "delta_rss_mb": 123.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.1461, + "full_recall_at_5": 0.0366, + "mean_recall_at_10": 0.2191, + "full_recall_at_10": 0.0366, + "mean_recall_at_20": 0.3053, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1187, + "mean_precision_at_10": 0.0488, + "zero_hit_rate": 0.439, + "median_tokens_delivered": 70828, + "median_latency_ms": 26.33, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.1461, + "full_recall_at_5": 0.0366, + "mean_recall_at_10": 0.2191, + "full_recall_at_10": 0.0366, + "mean_recall_at_20": 0.3053, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1187, + "mean_precision_at_10": 0.0488, + "zero_hit_rate": 0.439, + "median_tokens_delivered": 70828, + "median_latency_ms": 26.33, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.1344, + "full_recall_at_5": 0.0213, + "mean_recall_at_10": 0.2085, + "full_recall_at_10": 0.0213, + "mean_recall_at_20": 0.2986, + "full_recall_at_20": 0.0426, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1184, + "mean_precision_at_10": 0.0532, + "zero_hit_rate": 0.4043, + "median_tokens_delivered": 71582, + "median_latency_ms": 26.306, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.1619, + "full_recall_at_5": 0.0571, + "mean_recall_at_10": 0.2333, + "full_recall_at_10": 0.0571, + "mean_recall_at_20": 0.3143, + "full_recall_at_20": 0.1143, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.119, + "mean_precision_at_10": 0.0429, + "zero_hit_rate": 0.4857, + "median_tokens_delivered": 70715, + "median_latency_ms": 26.36, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 50019.5, + "peak_rss_mb": 50201.9, + "delta_rss_mb": 182.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "keyword-search", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.3955, + "full_recall_at_5": 0.2195, + "mean_recall_at_10": 0.5118, + "full_recall_at_10": 0.3171, + "mean_recall_at_20": 0.6276, + "full_recall_at_20": 0.439, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0396, + "mean_recall_at_16000_tokens": 0.2815, + "mean_precision_at_10": 0.1098, + "zero_hit_rate": 0.1951, + "median_tokens_delivered": 101796, + "median_latency_ms": 65.6965, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.3955, + "full_recall_at_5": 0.2195, + "mean_recall_at_10": 0.5118, + "full_recall_at_10": 0.3171, + "mean_recall_at_20": 0.6276, + "full_recall_at_20": 0.439, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0396, + "mean_recall_at_16000_tokens": 0.2815, + "mean_precision_at_10": 0.1098, + "zero_hit_rate": 0.1951, + "median_tokens_delivered": 101796, + "median_latency_ms": 65.6965, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.4262, + "full_recall_at_5": 0.234, + "mean_recall_at_10": 0.4908, + "full_recall_at_10": 0.2979, + "mean_recall_at_20": 0.6255, + "full_recall_at_20": 0.4468, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0372, + "mean_recall_at_16000_tokens": 0.2954, + "mean_precision_at_10": 0.1213, + "zero_hit_rate": 0.2128, + "median_tokens_delivered": 100748, + "median_latency_ms": 67.466, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.3543, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.54, + "full_recall_at_10": 0.3429, + "mean_recall_at_20": 0.6305, + "full_recall_at_20": 0.4286, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0429, + "mean_recall_at_16000_tokens": 0.2629, + "mean_precision_at_10": 0.0943, + "zero_hit_rate": 0.1714, + "median_tokens_delivered": 103945, + "median_latency_ms": 59.764, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 50176.6, + "peak_rss_mb": 50600.1, + "delta_rss_mb": 423.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "agent-default", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.4211, + "full_recall_at_5": 0.2805, + "mean_recall_at_10": 0.5494, + "full_recall_at_10": 0.378, + "mean_recall_at_20": 0.6783, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0852, + "mean_recall_at_16000_tokens": 0.3516, + "mean_precision_at_10": 0.1159, + "zero_hit_rate": 0.1707, + "median_tokens_delivered": 77547.5, + "median_latency_ms": 83.079, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.4211, + "full_recall_at_5": 0.2805, + "mean_recall_at_10": 0.5494, + "full_recall_at_10": 0.378, + "mean_recall_at_20": 0.6783, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0852, + "mean_recall_at_16000_tokens": 0.3516, + "mean_precision_at_10": 0.1159, + "zero_hit_rate": 0.1707, + "median_tokens_delivered": 77547.5, + "median_latency_ms": 83.079, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.5603, + "full_recall_at_5": 0.383, + "mean_recall_at_10": 0.6521, + "full_recall_at_10": 0.4681, + "mean_recall_at_20": 0.7351, + "full_recall_at_20": 0.5532, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1273, + "mean_recall_at_16000_tokens": 0.4922, + "mean_precision_at_10": 0.1468, + "zero_hit_rate": 0.1277, + "median_tokens_delivered": 77328, + "median_latency_ms": 91.155, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.2343, + "full_recall_at_5": 0.1429, + "mean_recall_at_10": 0.4114, + "full_recall_at_10": 0.2571, + "mean_recall_at_20": 0.6019, + "full_recall_at_20": 0.4286, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0286, + "mean_recall_at_16000_tokens": 0.1629, + "mean_precision_at_10": 0.0743, + "zero_hit_rate": 0.2286, + "median_tokens_delivered": 77740, + "median_latency_ms": 80.06, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 82 + }, + "resources": { + "baseline_rss_mb": 50286.2, + "peak_rss_mb": 50529.8, + "delta_rss_mb": 243.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.275, + "full_recall_at_5": 0.122, + "mean_recall_at_10": 0.3104, + "full_recall_at_10": 0.1585, + "mean_recall_at_20": 0.3539, + "full_recall_at_20": 0.1707, + "mean_recall_at_1000_tokens": 0.2606, + "mean_recall_at_4000_tokens": 0.3539, + "mean_recall_at_16000_tokens": 0.3539, + "mean_precision_at_10": 0.0687, + "zero_hit_rate": 0.4512, + "median_tokens_delivered": 1687, + "median_latency_ms": 2195.7945, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.275, + "full_recall_at_5": 0.122, + "mean_recall_at_10": 0.3104, + "full_recall_at_10": 0.1585, + "mean_recall_at_20": 0.3539, + "full_recall_at_20": 0.1707, + "mean_recall_at_1000_tokens": 0.2606, + "mean_recall_at_4000_tokens": 0.3539, + "mean_recall_at_16000_tokens": 0.3539, + "mean_precision_at_10": 0.0687, + "zero_hit_rate": 0.4512, + "median_tokens_delivered": 1687, + "median_latency_ms": 2195.7945, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.3557, + "full_recall_at_5": 0.1702, + "mean_recall_at_10": 0.3748, + "full_recall_at_10": 0.1702, + "mean_recall_at_20": 0.4145, + "full_recall_at_20": 0.1702, + "mean_recall_at_1000_tokens": 0.3589, + "mean_recall_at_4000_tokens": 0.4145, + "mean_recall_at_16000_tokens": 0.4145, + "mean_precision_at_10": 0.0871, + "zero_hit_rate": 0.3404, + "median_tokens_delivered": 1687, + "median_latency_ms": 2199.373, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.1667, + "full_recall_at_5": 0.0571, + "mean_recall_at_10": 0.2238, + "full_recall_at_10": 0.1429, + "mean_recall_at_20": 0.2724, + "full_recall_at_20": 0.1714, + "mean_recall_at_1000_tokens": 0.1286, + "mean_recall_at_4000_tokens": 0.2724, + "mean_recall_at_16000_tokens": 0.2724, + "mean_precision_at_10": 0.0441, + "zero_hit_rate": 0.6, + "median_tokens_delivered": 1694, + "median_latency_ms": 2190.61, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 81, + "indexed-but-returned-nothing": 1 + }, + "resources": { + "baseline_rss_mb": 49762.4, + "peak_rss_mb": 50257.3, + "delta_rss_mb": 494.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 191111168 + } + }, + { + "provider_id": "token-savior-regex", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.163, + "full_recall_at_5": 0.0732, + "mean_recall_at_10": 0.1976, + "full_recall_at_10": 0.0976, + "mean_recall_at_20": 0.2159, + "full_recall_at_20": 0.122, + "mean_recall_at_1000_tokens": 0.0356, + "mean_recall_at_4000_tokens": 0.2098, + "mean_recall_at_16000_tokens": 0.2159, + "mean_precision_at_10": 0.0886, + "zero_hit_rate": 0.6585, + "median_tokens_delivered": 3555, + "median_latency_ms": 1555.7565, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.163, + "full_recall_at_5": 0.0732, + "mean_recall_at_10": 0.1976, + "full_recall_at_10": 0.0976, + "mean_recall_at_20": 0.2159, + "full_recall_at_20": 0.122, + "mean_recall_at_1000_tokens": 0.0356, + "mean_recall_at_4000_tokens": 0.2098, + "mean_recall_at_16000_tokens": 0.2159, + "mean_precision_at_10": 0.0886, + "zero_hit_rate": 0.6585, + "median_tokens_delivered": 3555, + "median_latency_ms": 1555.7565, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.1922, + "full_recall_at_5": 0.0851, + "mean_recall_at_10": 0.2418, + "full_recall_at_10": 0.1277, + "mean_recall_at_20": 0.2525, + "full_recall_at_20": 0.1489, + "mean_recall_at_1000_tokens": 0.0337, + "mean_recall_at_4000_tokens": 0.2418, + "mean_recall_at_16000_tokens": 0.2525, + "mean_precision_at_10": 0.0992, + "zero_hit_rate": 0.5957, + "median_tokens_delivered": 4003, + "median_latency_ms": 1599.669, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.1238, + "full_recall_at_5": 0.0571, + "mean_recall_at_10": 0.1381, + "full_recall_at_10": 0.0571, + "mean_recall_at_20": 0.1667, + "full_recall_at_20": 0.0857, + "mean_recall_at_1000_tokens": 0.0381, + "mean_recall_at_4000_tokens": 0.1667, + "mean_recall_at_16000_tokens": 0.1667, + "mean_precision_at_10": 0.0744, + "zero_hit_rate": 0.7429, + "median_tokens_delivered": 2825, + "median_latency_ms": 1513.619, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 77, + "indexed-but-returned-nothing": 5 + }, + "resources": { + "baseline_rss_mb": 48505.7, + "peak_rss_mb": 49331.9, + "delta_rss_mb": 826.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "gortex", + "summary": { + "all": { + "cases": 82, + "mean_recall_at_5": 0.2128, + "full_recall_at_5": 0.0976, + "mean_recall_at_10": 0.2159, + "full_recall_at_10": 0.0976, + "mean_recall_at_20": 0.2159, + "full_recall_at_20": 0.0976, + "mean_recall_at_1000_tokens": 0.2128, + "mean_recall_at_4000_tokens": 0.2159, + "mean_recall_at_16000_tokens": 0.2159, + "mean_precision_at_10": 0.1643, + "zero_hit_rate": 0.6341, + "median_tokens_delivered": 437, + "median_latency_ms": 27891.853, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 82, + "mean_recall_at_5": 0.2128, + "full_recall_at_5": 0.0976, + "mean_recall_at_10": 0.2159, + "full_recall_at_10": 0.0976, + "mean_recall_at_20": 0.2159, + "full_recall_at_20": 0.0976, + "mean_recall_at_1000_tokens": 0.2128, + "mean_recall_at_4000_tokens": 0.2159, + "mean_recall_at_16000_tokens": 0.2159, + "mean_precision_at_10": 0.1643, + "zero_hit_rate": 0.6341, + "median_tokens_delivered": 437, + "median_latency_ms": 27891.853, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 47, + "mean_recall_at_5": 0.1862, + "full_recall_at_5": 0.0638, + "mean_recall_at_10": 0.1915, + "full_recall_at_10": 0.0638, + "mean_recall_at_20": 0.1915, + "full_recall_at_20": 0.0638, + "mean_recall_at_1000_tokens": 0.1862, + "mean_recall_at_4000_tokens": 0.1915, + "mean_recall_at_16000_tokens": 0.1915, + "mean_precision_at_10": 0.1383, + "zero_hit_rate": 0.6383, + "median_tokens_delivered": 449, + "median_latency_ms": 36142.634, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 35, + "mean_recall_at_5": 0.2486, + "full_recall_at_5": 0.1429, + "mean_recall_at_10": 0.2486, + "full_recall_at_10": 0.1429, + "mean_recall_at_20": 0.2486, + "full_recall_at_20": 0.1429, + "mean_recall_at_1000_tokens": 0.2486, + "mean_recall_at_4000_tokens": 0.2486, + "mean_recall_at_16000_tokens": 0.2486, + "mean_precision_at_10": 0.1991, + "zero_hit_rate": 0.6286, + "median_tokens_delivered": 384, + "median_latency_ms": 22642.103, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 53, + "indexed-but-returned-nothing": 29 + }, + "resources": { + "baseline_rss_mb": 49153.2, + "peak_rss_mb": 56008.3, + "delta_rss_mb": 6855, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55380, + "latency_ms": 19.348, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14735, + "latency_ms": 12.905, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33582, + "latency_ms": 11.978, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40619, + "latency_ms": 12.831, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34349, + "latency_ms": 12.132, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49787, + "latency_ms": 11.256, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19564, + "latency_ms": 11.049, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22900, + "latency_ms": 11.888, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [ + "test/types/request.test-d.ts" + ], + "missed": [ + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 29021, + "latency_ms": 11.314, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42213, + "latency_ms": 11.868, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25050, + "latency_ms": 11.038, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47319, + "latency_ms": 12.095, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 25876, + "latency_ms": 12.189, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 52479, + "latency_ms": 11.451, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [], + "missed": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14502, + "latency_ms": 11.244, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js" + ], + "missed": [ + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 15, + "tokens_delivered": 18095, + "latency_ms": 11.351, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29377, + "latency_ms": 11.525, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31292, + "latency_ms": 11.25, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45307, + "latency_ms": 11.52, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18048, + "latency_ms": 12.134, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28081, + "latency_ms": 12.816, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29389, + "latency_ms": 11.95, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49530, + "latency_ms": 10.601, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28931, + "latency_ms": 12.048, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23861, + "latency_ms": 12.202, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30703, + "latency_ms": 11.94, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28832, + "latency_ms": 10.781, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19751, + "latency_ms": 11.984, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 38242, + "latency_ms": 11.172, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33259, + "latency_ms": 10.66, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29737, + "latency_ms": 11.43, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30714, + "latency_ms": 11.021, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26147, + "latency_ms": 11.162, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23070, + "latency_ms": 12.203, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25310, + "latency_ms": 11.087, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/find-route.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33788, + "latency_ms": 10.576, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31982, + "latency_ms": 11.319, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10713, + "latency_ms": 11.882, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18276, + "latency_ms": 11.805, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30719, + "latency_ms": 11.862, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34315, + "latency_ms": 11.714, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42597, + "latency_ms": 11.202, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32783, + "latency_ms": 11.808, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/404s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14947, + "latency_ms": 12.798, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22479, + "latency_ms": 12.883, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30757, + "latency_ms": 11.459, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30239, + "latency_ms": 10.987, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18963, + "latency_ms": 11.689, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "test/hooks.test.js" + ], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 56605, + "latency_ms": 11.845, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29992, + "latency_ms": 11.35, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28153, + "latency_ms": 11.334, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30262, + "latency_ms": 11.626, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45885, + "latency_ms": 11.356, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23215, + "latency_ms": 11.945, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36564, + "latency_ms": 11.953, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46897, + "latency_ms": 11.704, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32459, + "latency_ms": 12.111, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22393, + "latency_ms": 11.635, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28957, + "latency_ms": 11.128, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51533, + "latency_ms": 12.19, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30177, + "latency_ms": 12.079, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42868, + "latency_ms": 11.796, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34695, + "latency_ms": 12.192, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40034, + "latency_ms": 11.62, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46434, + "latency_ms": 11.628, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36331, + "latency_ms": 10.987, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30307, + "latency_ms": 10.829, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40300, + "latency_ms": 11.946, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/plugin-utils.js" + ], + "missed": [ + "lib/errors.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 32313, + "latency_ms": 11.736, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/decorate.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 28910, + "latency_ms": 11.997, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43992, + "latency_ms": 11.602, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23870, + "latency_ms": 12.451, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42687, + "latency_ms": 12.832, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26329, + "latency_ms": 12.875, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28075, + "latency_ms": 12.185, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46278, + "latency_ms": 12.652, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38893, + "latency_ms": 12.506, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22541, + "latency_ms": 12.242, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45100, + "latency_ms": 11.126, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [], + "missed": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17721, + "latency_ms": 11.437, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26451, + "latency_ms": 11.355, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24066, + "latency_ms": 11.619, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70557, + "latency_ms": 26.723, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82986, + "latency_ms": 30.323, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "fastify.js", + "types/instance.d.ts" + ], + "missed": [ + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 70557, + "latency_ms": 26.582, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts" + ], + "missed": [ + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 70629, + "latency_ms": 25.738, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 87404, + "latency_ms": 26.417, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [ + "fastify.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 63466, + "latency_ms": 25.048, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70776, + "latency_ms": 25.876, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70662, + "latency_ms": 25.499, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69669, + "latency_ms": 26.477, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/request-error.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 66533, + "latency_ms": 25.592, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "fastify.js" + ], + "missed": [ + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 65423, + "latency_ms": 26.325, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 71928, + "latency_ms": 25.671, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 74650, + "latency_ms": 25.529, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70662, + "latency_ms": 26.484, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [], + "missed": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70557, + "latency_ms": 25.268, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "lib/route.js" + ], + "missed": [ + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 74728, + "latency_ms": 24.446, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69389, + "latency_ms": 24.851, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/http2/plain.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 71207, + "latency_ms": 24.427, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82986, + "latency_ms": 36.962, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64300, + "latency_ms": 25.647, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70546, + "latency_ms": 24.62, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 89027, + "latency_ms": 26.864, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [ + "fastify.js", + "lib/server.js" + ], + "missed": [ + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 82521, + "latency_ms": 26.131, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 80366, + "latency_ms": 28.689, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 74508, + "latency_ms": 25.383, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 64846, + "latency_ms": 25.492, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74429, + "latency_ms": 26.208, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/errors.js", + "test/web-api.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 84523, + "latency_ms": 27.033, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71253, + "latency_ms": 24.315, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 74448, + "latency_ms": 25.346, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 82525, + "latency_ms": 25.631, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62953, + "latency_ms": 25.208, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63778, + "latency_ms": 26.36, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 60591, + "latency_ms": 26.709, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63424, + "latency_ms": 28.096, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/find-route.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 63892, + "latency_ms": 25.979, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80545, + "latency_ms": 25.796, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 63909, + "latency_ms": 23.923, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68086, + "latency_ms": 24.108, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 84613, + "latency_ms": 24.828, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/reply.js", + "lib/request.js" + ], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 5, + "tokens_delivered": 63724, + "latency_ms": 25.616, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 70641, + "latency_ms": 25.193, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 61937, + "latency_ms": 35.202, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/404s.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 72316, + "latency_ms": 27.621, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "lib/route.js" + ], + "missed": [ + "test/hooks.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 63201, + "latency_ms": 26.48, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82525, + "latency_ms": 28.201, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64345, + "latency_ms": 27.247, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54068, + "latency_ms": 27.905, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 62953, + "latency_ms": 26.657, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/internals/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 69439, + "latency_ms": 28.044, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 74456, + "latency_ms": 26.565, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 68086, + "latency_ms": 27.87, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84659, + "latency_ms": 28.15, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70880, + "latency_ms": 26.238, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 70638, + "latency_ms": 25.894, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74650, + "latency_ms": 26.913, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63886, + "latency_ms": 28.612, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/fastify-instance.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 86006, + "latency_ms": 27.873, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70662, + "latency_ms": 26.626, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 84631, + "latency_ms": 26.335, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 89074, + "latency_ms": 27.824, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64345, + "latency_ms": 28.905, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 72779, + "latency_ms": 28.591, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [ + "test/types/instance.test-d.ts" + ], + "missed": [ + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 65418, + "latency_ms": 27.439, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74997, + "latency_ms": 28.622, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63778, + "latency_ms": 29.689, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 74705, + "latency_ms": 26.135, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 72798, + "latency_ms": 25.404, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js" + ], + "missed": [ + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 71582, + "latency_ms": 26.945, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/reply.js", + "lib/request.js" + ], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 5, + "tokens_delivered": 63025, + "latency_ms": 26.306, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 72300, + "latency_ms": 26.929, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 74694, + "latency_ms": 26.097, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64846, + "latency_ms": 26.092, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74728, + "latency_ms": 26.199, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 74702, + "latency_ms": 24.194, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 72267, + "latency_ms": 25.727, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 72267, + "latency_ms": 27.226, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 84982, + "latency_ms": 27.167, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 70715, + "latency_ms": 26.362, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "test/types/instance.test-d.ts" + ], + "missed": [ + "build/build-validation.js", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 65361, + "latency_ms": 25.45, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84594, + "latency_ms": 26.796, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 84607, + "latency_ms": 24.738, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [ + "test/reply-error.test.js" + ], + "missed": [ + "lib/hooks.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 122182, + "latency_ms": 77.778, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67691, + "latency_ms": 38.273, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "fastify.js", + "lib/warnings.js" + ], + "missed": [ + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 122295, + "latency_ms": 46.474, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 122718, + "latency_ms": 70.693, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "types/instance.d.ts" + ], + "missed": [ + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 67617, + "latency_ms": 37.262, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [ + "fastify.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 80557, + "latency_ms": 41.473, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 123787, + "latency_ms": 60.953, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 93462, + "latency_ms": 96.601, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 94851, + "latency_ms": 57.005, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [ + "lib/request.js", + "test/request-error.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 68051, + "latency_ms": 47.452, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "fastify.js" + ], + "missed": [ + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 112964, + "latency_ms": 38.245, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 114483, + "latency_ms": 75.148, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 62694, + "latency_ms": 73.383, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 117798, + "latency_ms": 79.787, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [ + "test/internals/reply.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 106900, + "latency_ms": 79.287, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "lib/errors.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 104642, + "latency_ms": 57.68, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46148, + "latency_ms": 45.246, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/http2/plain.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 100963, + "latency_ms": 58.097, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49265, + "latency_ms": 53.38, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/plugin-override.js" + ], + "missed": [ + "test/route-prefix.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 97637, + "latency_ms": 70.172, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69045, + "latency_ms": 58.76, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45900, + "latency_ms": 39.108, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [ + "fastify.js", + "lib/server.js", + "test/http2/closing.test.js" + ], + "missed": [ + "lib/symbols.js" + ], + "recall_at_5": 0.75, + "precision_at_5": 0.6, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 2, + "tokens_delivered": 61669, + "latency_ms": 47.446, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 107111, + "latency_ms": 70.199, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 110425, + "latency_ms": 29.196, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87171, + "latency_ms": 44.605, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 55573, + "latency_ms": 65.878, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [ + "lib/errors.js", + "lib/reply.js" + ], + "missed": [ + "test/web-api.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 111497, + "latency_ms": 57.527, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "fastify.js", + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 121245, + "latency_ms": 69.186, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 83432, + "latency_ms": 84.824, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 130386, + "latency_ms": 82.115, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 77995, + "latency_ms": 59.925, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [ + "test/helper.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 105097, + "latency_ms": 55.241, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [ + "fastify.js", + "lib/server.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 97321, + "latency_ms": 56.929, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [ + "test/close-pipelining.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 58970, + "latency_ms": 50.905, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js", + "test/find-route.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 100748, + "latency_ms": 51.712, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [ + "test/content-parser.test.js" + ], + "missed": [ + "lib/content-type-parser.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 73995, + "latency_ms": 26.243, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 64359, + "latency_ms": 50.135, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [ + "lib/handle-request.js" + ], + "missed": [ + "lib/error-handler.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 63246, + "latency_ms": 70.519, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 119274, + "latency_ms": 37.04, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/content-type-parser.js", + "lib/reply.js" + ], + "missed": [ + "lib/content-type.js", + "lib/handle-request.js", + "lib/request.js" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 99465, + "latency_ms": 78.258, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 95568, + "latency_ms": 57.402, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [ + "test/hooks-async.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 77375, + "latency_ms": 36.944, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [ + "test/404s.test.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 115276, + "latency_ms": 46.935, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 109415, + "latency_ms": 78.06, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 128881, + "latency_ms": 66.261, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [ + "types/logger.d.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 85813, + "latency_ms": 59.764, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 77052, + "latency_ms": 40.701, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 124179, + "latency_ms": 92.42, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 118496, + "latency_ms": 89.955, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 99496, + "latency_ms": 117.885, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 103945, + "latency_ms": 46.597, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 120633, + "latency_ms": 44.79, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [ + "lib/error-handler.js", + "lib/handle-request.js" + ], + "missed": [ + "lib/error-status.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 36837, + "latency_ms": 71.7, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js", + "test/close.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 86026, + "latency_ms": 54.911, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117282, + "latency_ms": 69.875, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 115269, + "latency_ms": 87.803, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 120178, + "latency_ms": 66.754, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [ + "types/logger.d.ts" + ], + "missed": [ + "lib/log-controller.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 132835, + "latency_ms": 80.129, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "missed": [], + "recall_at_5": 0.75, + "precision_at_5": 0.6, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 53001, + "latency_ms": 57.63, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "missed": [ + "lib/error-handler.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 139650, + "latency_ms": 140.414, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [ + "lib/handleRequest.js" + ], + "missed": [ + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 78858, + "latency_ms": 74.134, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/router-options.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 114663, + "latency_ms": 77.915, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86593, + "latency_ms": 75.617, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 99276, + "latency_ms": 71.108, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/contentTypeParser.js", + "lib/errors.js" + ], + "missed": [ + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 111702, + "latency_ms": 41.245, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 102629, + "latency_ms": 98.838, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [ + "test/types/route.test-d.ts" + ], + "missed": [ + "types/route.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 123063, + "latency_ms": 77.403, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js" + ], + "missed": [ + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0.6, + "precision_at_5": 0.6, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 1, + "tokens_delivered": 124894, + "latency_ms": 92.058, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/decorate.js", + "lib/reply.js" + ], + "missed": [ + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 119132, + "latency_ms": 105.151, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [ + "lib/validation.js" + ], + "missed": [ + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 140649, + "latency_ms": 94.766, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 111682, + "latency_ms": 69.703, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53367, + "latency_ms": 39.384, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [ + "lib/schema-controller.js" + ], + "missed": [ + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 126553, + "latency_ms": 74.179, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/throw.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 85444, + "latency_ms": 68.53, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [ + "lib/head-route.js", + "test/route.6.test.js" + ], + "missed": [ + "test/route.7.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 120657, + "latency_ms": 59.376, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [ + "lib/request.js", + "test/server.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 62516, + "latency_ms": 39.212, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71679, + "latency_ms": 67.466, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 132990, + "latency_ms": 38.373, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "missed": [ + "test/issue-4959.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.8333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 1, + "tokens_delivered": 108175, + "latency_ms": 76.539, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 97985, + "latency_ms": 73.507, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js", + "test/close.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 136620, + "latency_ms": 65.515, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 102517, + "latency_ms": 115.424, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68265, + "latency_ms": 64.102, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js" + ], + "missed": [ + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 71619, + "latency_ms": 73.077, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 68514, + "latency_ms": 97.186, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41308, + "latency_ms": 61.012, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [ + "fastify.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 68614, + "latency_ms": 63.723, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 91220, + "latency_ms": 82.316, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 60261, + "latency_ms": 123.402, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63531, + "latency_ms": 84.869, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [ + "lib/request.js", + "test/request-error.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 46499, + "latency_ms": 71.591, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [ + "fastify.d.ts", + "fastify.js" + ], + "missed": [ + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 57763, + "latency_ms": 65.346, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "missed": [ + "lib/errors.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 106419, + "latency_ms": 99.677, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 59825, + "latency_ms": 87.044, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 117798, + "latency_ms": 98.737, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [ + "test/internals/reply.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 79907, + "latency_ms": 99.705, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "lib/errors.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 77328, + "latency_ms": 81.42, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46148, + "latency_ms": 70.547, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/http2/plain.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 73891, + "latency_ms": 83.037, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49652, + "latency_ms": 71.855, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/plugin-override.js" + ], + "missed": [ + "test/route-prefix.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 97637, + "latency_ms": 83.121, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 77355, + "latency_ms": 84.108, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 47963, + "latency_ms": 60.56, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [ + "fastify.js", + "lib/server.js", + "test/http2/closing.test.js" + ], + "missed": [ + "lib/symbols.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 40107, + "latency_ms": 69.96, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 46713, + "latency_ms": 114.096, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 110425, + "latency_ms": 50.552, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87171, + "latency_ms": 67.67, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 55573, + "latency_ms": 86.176, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [ + "lib/errors.js", + "lib/reply.js" + ], + "missed": [ + "test/web-api.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 89624, + "latency_ms": 79.414, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "fastify.js", + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 121245, + "latency_ms": 81.275, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 83432, + "latency_ms": 99.878, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 54793, + "latency_ms": 103.188, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 77001, + "latency_ms": 80.72, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [ + "test/helper.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 100428, + "latency_ms": 78.37, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [ + "fastify.js", + "lib/server.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 89200, + "latency_ms": 81.882, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [ + "test/close-pipelining.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 39536, + "latency_ms": 75.898, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js", + "test/find-route.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 81816, + "latency_ms": 69.231, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [ + "test/content-parser.test.js" + ], + "missed": [ + "lib/content-type-parser.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 73995, + "latency_ms": 42.525, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 44503, + "latency_ms": 69.173, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [ + "lib/error-handler.js", + "lib/handle-request.js" + ], + "missed": [ + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 50298, + "latency_ms": 94.275, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 61252, + "latency_ms": 57.485, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/reply.js" + ], + "missed": [ + "lib/handle-request.js", + "lib/request.js" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 1, + "tokens_delivered": 91011, + "latency_ms": 100.499, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 95568, + "latency_ms": 80.167, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [ + "test/hooks-async.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 66184, + "latency_ms": 59.785, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [ + "test/404s.test.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 115276, + "latency_ms": 67.904, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 94135, + "latency_ms": 105.075, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 98501, + "latency_ms": 94.216, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [ + "types/logger.d.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 44113, + "latency_ms": 80.06, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 81366, + "latency_ms": 62.954, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 95197, + "latency_ms": 114.377, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 122034, + "latency_ms": 111.648, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 96582, + "latency_ms": 147.452, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 103945, + "latency_ms": 65.345, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 120633, + "latency_ms": 59.87, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [ + "lib/error-handler.js", + "lib/handle-request.js" + ], + "missed": [ + "lib/error-status.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 10, + "tokens_delivered": 25772, + "latency_ms": 92.55, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js", + "test/close.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 86026, + "latency_ms": 80.617, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 107750, + "latency_ms": 93.656, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 67776, + "latency_ms": 119.454, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 120178, + "latency_ms": 95.921, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [ + "types/logger.d.ts" + ], + "missed": [ + "lib/log-controller.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 98863, + "latency_ms": 100.771, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "missed": [], + "recall_at_5": 0.75, + "precision_at_5": 0.6, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 53001, + "latency_ms": 86.385, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "lib/error-handler.js", + "types/instance.d.ts" + ], + "missed": [ + "test/500s.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 82352, + "latency_ms": 163.455, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [ + "lib/handleRequest.js" + ], + "missed": [ + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 59579, + "latency_ms": 68.108, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [ + "lib/route.js", + "test/router-options.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 84831, + "latency_ms": 100.797, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55821, + "latency_ms": 97.935, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [ + "test/schema-serialization.test.js" + ], + "missed": [ + "lib/schemas.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 81552, + "latency_ms": 99.787, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/contentTypeParser.js", + "lib/errors.js" + ], + "missed": [ + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 100458, + "latency_ms": 62.451, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 102629, + "latency_ms": 111.854, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 99867, + "latency_ms": 88.935, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js" + ], + "missed": [ + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 71484, + "latency_ms": 109.986, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js" + ], + "missed": [ + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 86112, + "latency_ms": 112.865, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 85538, + "latency_ms": 105.967, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 77125, + "latency_ms": 89.487, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53367, + "latency_ms": 51.94, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [ + "lib/schema-controller.js" + ], + "missed": [ + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 84719, + "latency_ms": 82.675, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 77740, + "latency_ms": 75.778, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 59083, + "latency_ms": 78.072, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [ + "lib/request.js", + "test/server.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 62516, + "latency_ms": 49.318, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 45381, + "latency_ms": 91.155, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 118359, + "latency_ms": 66.395, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "missed": [ + "test/issue-4959.test.js" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.8333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 3, + "tokens_delivered": 69923, + "latency_ms": 104.022, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 65547, + "latency_ms": 91.265, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js", + "test/close.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 138562, + "latency_ms": 86.738, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1678, + "latency_ms": 2554.639, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1661, + "latency_ms": 2144.995, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "fastify.js", + "lib/warnings.js" + ], + "missed": [ + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.4, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 1, + "tokens_delivered": 1681, + "latency_ms": 2227.664, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4561, + "latency_ms": 2263.939, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 766, + "latency_ms": 2218.374, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [ + "fastify.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 6521, + "latency_ms": 2315.204, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 189, + "latency_ms": 2442.086, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js" + ], + "missed": [ + "test/content-parser.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1690, + "latency_ms": 2365.077, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1687, + "latency_ms": 2186.592, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/request-error.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1670, + "latency_ms": 2158.607, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "fastify.d.ts", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 2459, + "latency_ms": 2126.767, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js", + "lib/errors.js" + ], + "missed": [ + "test/content-parser.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 1799, + "latency_ms": 2342.613, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 8141, + "latency_ms": 2234.346, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "test/decorator-instance-properties.test.js" + ], + "missed": [ + "lib/decorate.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 173, + "latency_ms": 2317.294, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [ + "test/internals/reply.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 220, + "latency_ms": 2279.757, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "lib/route.js" + ], + "missed": [ + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3308, + "latency_ms": 2244.706, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6174, + "latency_ms": 2249.682, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3395, + "latency_ms": 2190.61, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 147, + "latency_ms": 2145.626, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1330, + "latency_ms": 2253.402, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1500, + "latency_ms": 2300.591, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1667, + "latency_ms": 2224.45, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1188, + "latency_ms": 2132.358, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1679, + "latency_ms": 2171.006, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5053, + "latency_ms": 2209.776, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1671, + "latency_ms": 2148.595, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7, + "latency_ms": 2220.477, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 960, + "latency_ms": 2171.025, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "fastify.js", + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1694, + "latency_ms": 2231.054, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "test/trust-proxy.test.js" + ], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1687, + "latency_ms": 2205.209, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 14, + "tokens_delivered": 1680, + "latency_ms": 2149.391, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1371, + "latency_ms": 2199.373, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [ + "test/helper.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 3041, + "latency_ms": 2149.168, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [ + "fastify.js", + "lib/server.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1696, + "latency_ms": 2158.551, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [ + "test/close-pipelining.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1657, + "latency_ms": 2145.23, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/find-route.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 6029, + "latency_ms": 2235.056, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5696, + "latency_ms": 2159.232, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1695, + "latency_ms": 2271.731, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [ + "lib/error-handler.js", + "lib/symbols.js" + ], + "missed": [ + "lib/handle-request.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 1704, + "latency_ms": 2206.943, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/route-prefix.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 2945, + "latency_ms": 2225.687, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/reply.js" + ], + "missed": [ + "lib/handle-request.js", + "lib/request.js" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 1, + "tokens_delivered": 1674, + "latency_ms": 2283.751, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1682, + "latency_ms": 2280.806, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [ + "test/hooks-async.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1674, + "latency_ms": 2147.099, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/404s.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2855, + "latency_ms": 2090.837, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "test/hooks.test.js" + ], + "missed": [ + "lib/reply.js", + "lib/route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 12, + "tokens_delivered": 1677, + "latency_ms": 2169.91, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [ + "types/schema.d.ts" + ], + "missed": [ + "test/types/schema.test-d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2902, + "latency_ms": 2104.792, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1293, + "latency_ms": 2150.471, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1657, + "latency_ms": 2087.126, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "test/hooks.test.js" + ], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 1683, + "latency_ms": 2318.263, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/internals/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4652, + "latency_ms": 2150.702, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 1689, + "latency_ms": 2173.965, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1675, + "latency_ms": 2176.035, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5116, + "latency_ms": 2195.505, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4478, + "latency_ms": 2090.099, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2709, + "latency_ms": 2200.12, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [ + "types/request.d.ts" + ], + "missed": [ + "test/types/request.test-d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 215, + "latency_ms": 2229.821, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4323, + "latency_ms": 2065.471, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/fastify-instance.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 4025, + "latency_ms": 2066.551, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [ + "lib/log-controller.js" + ], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 4376, + "latency_ms": 2196.084, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js" + ], + "missed": [ + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 1674, + "latency_ms": 2906.779, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 391, + "latency_ms": 3619.947, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1683, + "latency_ms": 2278.558, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/router-options.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4577, + "latency_ms": 2152.813, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1668, + "latency_ms": 2168.476, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7404, + "latency_ms": 2207.458, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js" + ], + "missed": [ + "lib/contentTypeParser.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 15, + "tokens_delivered": 1696, + "latency_ms": 2104.595, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 233, + "latency_ms": 2194.729, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1686, + "latency_ms": 2152.403, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "lib/errors.js", + "lib/plugin-utils.js" + ], + "missed": [ + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 2, + "tokens_delivered": 1702, + "latency_ms": 2211.606, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 201, + "latency_ms": 2186.8, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [ + "test/validation-error-handling.test.js" + ], + "missed": [ + "lib/validation.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1717, + "latency_ms": 2116.59, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1670, + "latency_ms": 2267.673, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5585, + "latency_ms": 2159.192, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6131, + "latency_ms": 2218.143, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3578, + "latency_ms": 2206.135, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [ + "test/route.6.test.js" + ], + "missed": [ + "lib/head-route.js", + "test/route.7.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 1665, + "latency_ms": 2129.151, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3795, + "latency_ms": 2120.156, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1688, + "latency_ms": 2145.133, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3331, + "latency_ms": 2202.243, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [], + "missed": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5412, + "latency_ms": 2111.796, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1681, + "latency_ms": 2121.301, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "test/close.test.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1671, + "latency_ms": 2204.407, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4381, + "latency_ms": 2129.652, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1908, + "latency_ms": 1013.005, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3574, + "latency_ms": 1287.906, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4309, + "latency_ms": 1732.508, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2576, + "latency_ms": 922.637, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1913, + "latency_ms": 1112.854, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [], + "missed": [ + "lib/content-type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3536, + "latency_ms": 1590.857, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4695, + "latency_ms": 2340.674, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3853, + "latency_ms": 1429.794, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [ + "lib/request.js", + "test/request-error.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2416, + "latency_ms": 1222.78, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2216, + "latency_ms": 1019.612, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js" + ], + "missed": [ + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 4443, + "latency_ms": 2005.499, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3807, + "latency_ms": 1810.333, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [ + "test/decorator-instance-properties.test.js" + ], + "missed": [ + "lib/decorate.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4114, + "latency_ms": 2048.797, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [ + "test/internals/reply.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 4413, + "latency_ms": 2003.254, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3643, + "latency_ms": 1528.848, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1545, + "latency_ms": 1234.936, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/http2/plain.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3463, + "latency_ms": 1471.238, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2849, + "latency_ms": 1112.642, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 3055, + "latency_ms": 1784.512, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3199, + "latency_ms": 1529.042, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1651, + "latency_ms": 1013.238, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2269, + "latency_ms": 1207.659, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4280, + "latency_ms": 1436.391, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1378, + "latency_ms": 781.323, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1512, + "latency_ms": 1283.496, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1778, + "latency_ms": 1783.726, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/errors.js", + "test/web-api.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1429, + "recall_at_20": 0.3333, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 2351, + "latency_ms": 1568.089, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3397, + "latency_ms": 1607.03, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 4937, + "latency_ms": 1975.787, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [], + "missed": [ + "lib/server.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5397, + "latency_ms": 1910.723, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 2520, + "latency_ms": 1525.127, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2401, + "latency_ms": 1513.619, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3646, + "latency_ms": 1403.714, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [ + "test/close-pipelining.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1483, + "latency_ms": 1225.146, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/find-route.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3366, + "latency_ms": 1298.17, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 813, + "latency_ms": 763.058, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3251, + "latency_ms": 1224.114, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [ + "lib/handle-request.js" + ], + "missed": [ + "lib/error-handler.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1667, + "recall_at_20": 0.25, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 4, + "tokens_delivered": 4003, + "latency_ms": 1777.839, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2057, + "latency_ms": 1022.197, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/request.js" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.25, + "recall_at_10": 0.2, + "precision_at_10": 0.25, + "recall_at_20": 0.2, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 4823, + "latency_ms": 1968.203, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2970, + "latency_ms": 1537.528, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1583, + "latency_ms": 1081.846, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/404s.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2213, + "latency_ms": 1235.813, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.4286, + "recall_at_20": 1, + "precision_at_20": 0.4286, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5100, + "latency_ms": 2017.514, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4489, + "latency_ms": 1715.333, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2825, + "latency_ms": 1409.541, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2090, + "latency_ms": 952.024, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5464, + "latency_ms": 2316.419, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4943, + "latency_ms": 2162.793, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5183, + "latency_ms": 2227.378, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2299, + "latency_ms": 1286.964, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2560, + "latency_ms": 1307.388, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [ + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4306, + "latency_ms": 1694.706, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "fastify.js", + "test/close.test.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3580, + "latency_ms": 1543.424, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4356, + "latency_ms": 1731.552, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4943, + "latency_ms": 2047.391, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/fastify-instance.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 3252, + "latency_ms": 1763.22, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [ + "lib/log-controller.js" + ], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 5497, + "latency_ms": 1975.309, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "fastify.d.ts", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.2, + "recall_at_20": 0.25, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 3148, + "latency_ms": 1524.656, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5453, + "latency_ms": 2137.314, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2731, + "latency_ms": 1099.053, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/router-options.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 4651, + "latency_ms": 1670.633, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4416, + "latency_ms": 1589.2, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4490, + "latency_ms": 1605.131, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2187, + "latency_ms": 979.524, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 9, + "tokens_delivered": 6504, + "latency_ms": 2300.069, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4448, + "latency_ms": 1808.761, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5498, + "latency_ms": 2126.1, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4798, + "latency_ms": 2167.959, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5022, + "latency_ms": 2237.695, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4037, + "latency_ms": 1827.083, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1332, + "latency_ms": 1067.236, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4539, + "latency_ms": 1786.149, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/throw.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3763, + "latency_ms": 1464.166, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3809, + "latency_ms": 1347.436, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2590, + "latency_ms": 1042.171, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3897, + "latency_ms": 1599.669, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2437, + "latency_ms": 964.26, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [], + "missed": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4936, + "latency_ms": 1952.591, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2828, + "latency_ms": 1706.313, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3379, + "latency_ms": 1647.359, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-01ca8393b9d1", + "path_leak": false, + "required_files": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "found": [], + "missed": [ + "lib/hooks.js", + "test/reply-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2986, + "latency_ms": 5888.335, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-0dfa46c46687", + "path_leak": false, + "required_files": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "found": [], + "missed": [ + "lib/wrapThenable.js", + "test/issue-4959.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1869, + "latency_ms": 4664.232, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-0e9a3b35d8b1", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "lib/warnings.js", + "test/http-methods/custom-http-methods.test.js", + "test/types/instance.tst.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.3333, + "recall_at_10": 0.2, + "precision_at_10": 0.3333, + "recall_at_20": 0.2, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 1317, + "latency_ms": 13480.862, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-0f54a40f63b3", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2193, + "latency_ms": 6868.256, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-120661fab3b2", + "path_leak": false, + "required_files": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 384, + "latency_ms": 6404.025, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-153d78aaf952", + "path_leak": true, + "required_files": [ + "fastify.js" + ], + "found": [ + "fastify.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 449, + "latency_ms": 7834.347, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-15c2fb193346", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 749, + "latency_ms": 7748.536, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-16a74e7cb2d1", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "found": [ + "lib/content-type-parser.js", + "test/content-parser.test.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1905, + "latency_ms": 17746.902, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-16b2e4d01888", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 797, + "latency_ms": 11320.685, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-16c2c3db5cf0", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-error.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-error.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 840, + "latency_ms": 24188.043, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-1d5f8ae199ed", + "path_leak": false, + "required_files": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "fastify.js", + "test/types/fastify.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 754, + "latency_ms": 19151.287, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-1e0be1294e85", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "lib/errors.js", + "test/content-parser.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 438, + "latency_ms": 8398.3, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-21b4c3c10103", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 314, + "latency_ms": 12694.882, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-27938ac777c1", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1435, + "latency_ms": 26461.505, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-29bdcbb91f13", + "path_leak": true, + "required_files": [ + "lib/content-type.js", + "test/internals/reply.test.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [ + "test/internals/reply.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 421, + "latency_ms": 39995.061, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-2ae3e1d73491", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/route.js", + "test/logger/options.test.js", + "types/errors.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42, + "latency_ms": 55736.945, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-2af83d64b53a", + "path_leak": false, + "required_files": [ + "LICENSE" + ], + "found": [], + "missed": [ + "LICENSE" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 229, + "latency_ms": 19339.747, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-2bf4265b991a", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/http2/plain.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38, + "latency_ms": 33209.38, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-2c0be02b396b", + "path_leak": true, + "required_files": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/type-provider.test-d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 214, + "latency_ms": 36142.634, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-2f597a9297a3", + "path_leak": false, + "required_files": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/plugin-override.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 624, + "latency_ms": 20197.071, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-325682c9e5ab", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-6411.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41, + "latency_ms": 81520.151, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-329e40f09f23", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/reply-web-stream-locked.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36, + "latency_ms": 58008.42, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-36498f85a91c", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "lib/server.js", + "lib/symbols.js", + "test/http2/closing.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 240, + "latency_ms": 67225.355, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-37c7ca89d88f", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [], + "missed": [ + "lib/contentTypeParser.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43, + "latency_ms": 69864.41, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-39f0f24233cf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/request-port.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/request-port.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34, + "latency_ms": 80165.442, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-39f35f9a0596", + "path_leak": false, + "required_files": [ + "lib/warnings.js" + ], + "found": [ + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 413, + "latency_ms": 138922.786, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-3b0f76993d51", + "path_leak": false, + "required_files": [ + "lib/content-type.js" + ], + "found": [ + "lib/content-type.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 312, + "latency_ms": 244727.477, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-3bcb8cfb2e5c", + "path_leak": false, + "required_files": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "found": [], + "missed": [ + "lib/errors.js", + "lib/reply.js", + "test/web-api.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 247, + "latency_ms": 108324.498, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-47670c1fced0", + "path_leak": false, + "required_files": [ + "fastify.js", + "lib/warnings.js" + ], + "found": [ + "fastify.js", + "lib/warnings.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1818, + "latency_ms": 18030.895, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-4e1db5bd0012", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/trust-proxy.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 709, + "latency_ms": 20622.381, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-4e5a3631ec73", + "path_leak": false, + "required_files": [ + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 841, + "latency_ms": 12725.413, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-4fceec019ba2", + "path_leak": true, + "required_files": [ + "lib/warnings.js" + ], + "found": [], + "missed": [ + "lib/warnings.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 866, + "latency_ms": 47337.185, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-4ff4c1296267", + "path_leak": false, + "required_files": [ + "test/helper.js" + ], + "found": [], + "missed": [ + "test/helper.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 803, + "latency_ms": 6624.002, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-55345987cb51", + "path_leak": true, + "required_files": [ + "fastify.js", + "lib/server.js" + ], + "found": [ + "lib/server.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1528, + "latency_ms": 6213.5, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-55c4841507bd", + "path_leak": true, + "required_files": [ + "test/close-pipelining.test.js" + ], + "found": [], + "missed": [ + "test/close-pipelining.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 343, + "latency_ms": 24026.675, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-6682c4f9a76c", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/find-route.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/find-route.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1009, + "latency_ms": 12171.48, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-6a428d94f6bf", + "path_leak": false, + "required_files": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/content-type-parser.js", + "test/content-parser.test.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34, + "latency_ms": 61500.694, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-6ac2e9537053", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 400, + "latency_ms": 73305.016, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-6d202c6203f3", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/handle-request.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "found": [ + "lib/handle-request.js" + ], + "missed": [ + "lib/error-handler.js", + "lib/symbols.js", + "test/diagnostics-channel/async-error-handler.test.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 4, + "tokens_delivered": 1756, + "latency_ms": 27612.453, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-6e95cb9f6de4", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/route-prefix.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37, + "latency_ms": 67062.764, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-6f63ce9324b7", + "path_leak": true, + "required_files": [ + "lib/content-type-parser.js", + "lib/content-type.js", + "lib/handle-request.js", + "lib/reply.js", + "lib/request.js" + ], + "found": [ + "lib/content-type-parser.js", + "lib/handle-request.js", + "lib/reply.js" + ], + "missed": [ + "lib/content-type.js", + "lib/request.js" + ], + "recall_at_5": 0.6, + "precision_at_5": 0.6, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 1, + "tokens_delivered": 2272, + "latency_ms": 14737.671, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-7299a57d3fdc", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1368, + "latency_ms": 32245.635, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-750976cc4cb0", + "path_leak": true, + "required_files": [ + "test/hooks-async.test.js" + ], + "found": [], + "missed": [ + "test/hooks-async.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34, + "latency_ms": 41351.352, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-79ab8e956482", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/404s.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/404s.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 653, + "latency_ms": 27173.185, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-7f6d31980632", + "path_leak": true, + "required_files": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "lib/route.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1293, + "latency_ms": 82260.275, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-80ef70c2541c", + "path_leak": true, + "required_files": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "found": [], + "missed": [ + "test/types/schema.test-d.ts", + "types/schema.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44, + "latency_ms": 65935.776, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-812ef58f8ab1", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39, + "latency_ms": 148386.227, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-8435cb10f93e", + "path_leak": false, + "required_files": [ + "test/http2/secure-with-fallback.test.js" + ], + "found": [], + "missed": [ + "test/http2/secure-with-fallback.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35, + "latency_ms": 80623.127, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-8b9c07b645a8", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "found": [], + "missed": [ + "lib/reply.js", + "test/hooks.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 563, + "latency_ms": 28171.253, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-8ddd4fa85629", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/internals/hooks.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/internals/hooks.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1802, + "latency_ms": 181393.474, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-8dee9be05ebf", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/trust-proxy.test.js" + ], + "found": [ + "test/trust-proxy.test.js" + ], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 847, + "latency_ms": 12794.45, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-9026164f5a2e", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 898, + "latency_ms": 7482.474, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-9493c0fb716c", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/fix-root-primitive-coercion.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 258, + "latency_ms": 96589.069, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-970c57583252", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "found": [], + "missed": [ + "lib/error-handler.js", + "lib/error-status.js", + "lib/handle-request.js", + "lib/wrap-thenable.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2431, + "latency_ms": 42512.399, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-9ba6d6016fa9", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 357, + "latency_ms": 10874.562, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-9bbf60978d01", + "path_leak": true, + "required_files": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "found": [], + "missed": [ + "test/types/request.test-d.ts", + "types/request.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44, + "latency_ms": 7007.981, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-9be9fc07b769", + "path_leak": true, + "required_files": [ + "lib/contentTypeParser.js" + ], + "found": [ + "lib/contentTypeParser.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 664, + "latency_ms": 77580.827, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-9dc6486b6594", + "path_leak": false, + "required_files": [ + "fastify.js", + "test/fastify-instance.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/fastify-instance.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1242, + "latency_ms": 6442.479, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-9eaef5123b88", + "path_leak": false, + "required_files": [ + "lib/log-controller.js", + "types/logger.d.ts" + ], + "found": [ + "lib/log-controller.js" + ], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 863, + "latency_ms": 22642.103, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-af079bd4c60c", + "path_leak": true, + "required_files": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "found": [], + "missed": [ + "fastify.d.ts", + "lib/request.js", + "test/trust-proxy.test.js", + "test/types/fastify.tst.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 243, + "latency_ms": 75490.773, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-b03d0783f246", + "path_leak": true, + "required_files": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts", + "types/instance.d.ts" + ], + "found": [ + "types/instance.d.ts" + ], + "missed": [ + "lib/error-handler.js", + "test/500s.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 10, + "tokens_delivered": 3765, + "latency_ms": 10086.438, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-b0e255a2cdab", + "path_leak": false, + "required_files": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "found": [], + "missed": [ + "lib/handleRequest.js", + "test/internals/handle-request.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37, + "latency_ms": 49604.883, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-b4db85b665ac", + "path_leak": true, + "required_files": [ + "lib/route.js", + "test/router-options.test.js" + ], + "found": [ + "lib/route.js" + ], + "missed": [ + "test/router-options.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 722, + "latency_ms": 80962.198, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-c02c5f5696c8", + "path_leak": true, + "required_files": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "found": [], + "missed": [ + "test/types/instance.test-d.ts", + "test/types/reply.test-d.ts", + "test/types/type-provider.test-d.ts", + "types/reply.d.ts", + "types/type-provider.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43, + "latency_ms": 24542.825, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-c06fc06d33e6", + "path_leak": true, + "required_files": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "found": [], + "missed": [ + "lib/schemas.js", + "test/schema-serialization.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42, + "latency_ms": 45865.047, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-c277b9f0ec27", + "path_leak": false, + "required_files": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "types/errors.d.ts" + ], + "missed": [ + "lib/contentTypeParser.js", + "lib/errors.js", + "test/buffer.test.js", + "test/types/errors.test-d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 1, + "recall_at_10": 0.2, + "precision_at_10": 1, + "recall_at_20": 0.2, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 235, + "latency_ms": 123485.033, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-c2ba18c17769", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "found": [], + "missed": [ + "fastify.js", + "test/close-pipelining.test.js", + "test/custom-http-server.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45, + "latency_ms": 53387.582, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-c4a6e78e5d77", + "path_leak": true, + "required_files": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "found": [], + "missed": [ + "test/types/route.test-d.ts", + "types/route.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 382, + "latency_ms": 7382.128, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-c647de44ccea", + "path_leak": true, + "required_files": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts", + "types/errors.d.ts" + ], + "found": [ + "types/errors.d.ts" + ], + "missed": [ + "lib/errors.js", + "lib/plugin-utils.js", + "test/internals/plugin.test.js", + "test/types/errors.test-d.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1429, + "recall_at_20": 0.2, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 650, + "latency_ms": 22659.183, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-cc8d9a3c961d", + "path_leak": true, + "required_files": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "found": [], + "missed": [ + "lib/decorate.js", + "lib/reply.js", + "lib/request.js", + "test/decorator-instance-properties.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1434, + "latency_ms": 61765.206, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-d338dca5ab24", + "path_leak": true, + "required_files": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/validation-error-handling.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49, + "latency_ms": 196175.955, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-d33f017ffef3", + "path_leak": false, + "required_files": [ + "lib/reply.js", + "test/reply-trailers.test.js" + ], + "found": [ + "lib/reply.js" + ], + "missed": [ + "test/reply-trailers.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 738, + "latency_ms": 11855.811, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-d44b3f7b2d2e", + "path_leak": false, + "required_files": [ + "types/logger.d.ts" + ], + "found": [], + "missed": [ + "types/logger.d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 290, + "latency_ms": 84900.862, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-d76dbcd58bf6", + "path_leak": false, + "required_files": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "found": [], + "missed": [ + "lib/schema-controller.js", + "test/internals/schema-controller-perf.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 301, + "latency_ms": 11593.414, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-d9659819fbbc", + "path_leak": false, + "required_files": [ + "lib/route.js", + "test/throw.test.js" + ], + "found": [], + "missed": [ + "lib/route.js", + "test/throw.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 362, + "latency_ms": 180925.377, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-dd02e428ddbf", + "path_leak": true, + "required_files": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "found": [], + "missed": [ + "lib/head-route.js", + "test/route.6.test.js", + "test/route.7.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3067, + "latency_ms": 65782.44, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-e1aee4b872aa", + "path_leak": false, + "required_files": [ + "lib/request.js", + "test/server.test.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [ + "test/server.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 252, + "latency_ms": 24523.551, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-e1f58aba76e0", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/child-logger-factory.test.js" + ], + "found": [ + "fastify.js" + ], + "missed": [ + "test/child-logger-factory.test.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 9118, + "latency_ms": 54090.37, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-e4ffc205328d", + "path_leak": false, + "required_files": [ + "lib/reply.js" + ], + "found": [], + "missed": [ + "lib/reply.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37, + "latency_ms": 13977.796, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-f18cda12d62a", + "path_leak": true, + "required_files": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "found": [], + "missed": [ + "build/build-validation.js", + "fastify.d.ts", + "lib/config-validator.js", + "test/issue-4959.test.js", + "test/router-options.test.js", + "test/types/instance.test-d.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 900, + "latency_ms": 36004.717, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-f3d2bcb3963c", + "path_leak": false, + "required_files": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "found": [], + "missed": [ + "lib/validation.js", + "test/schema-validation.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 367, + "latency_ms": 82191.06, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "fastify-f5ef34443c5b", + "path_leak": true, + "required_files": [ + "fastify.js", + "test/close.test.js" + ], + "found": [ + "test/close.test.js" + ], + "missed": [ + "fastify.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 436, + "latency_ms": 22431.719, + "indexed_revision_matches": true + } + ], + "tier": "medium", + "tier_code_files": 300, + "provenance": "Scored before tier stamping and the per-call timeout were added; reused because every case completed (0 unavailable), so neither change could alter it." +} diff --git a/benchmarks/context-retrieval/results/T2-hugo.json b/benchmarks/context-retrieval/results/T2-hugo.json new file mode 100644 index 00000000..2164feba --- /dev/null +++ b/benchmarks/context-retrieval/results/T2-hugo.json @@ -0,0 +1,6890 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "hugo", + "head": "7b5199fdeff7d7682e9f979f468d55bc7b67b934" + }, + "tier": "medium", + "tier_code_files": 932, + "corpus_counts": { + "cases": 29, + "path_leak": 13, + "baseline_blind": 16, + "multi_file": 25 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.3009, + "control": 0, + "reason": null + }, + "needs_raw_payload_check": [ + { + "provider_id": "keyword-search", + "why": "scored exactly zero — read the raw payload before believing it" + } + ], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "Fix potential content XSS by escaping dangerous URLs in links and images", + "revision": "81a5cdca0788ca39574a17d444c9db29d0b19e27", + "returned": [ + "markup/goldmark/images/images_integration_test.go", + "markup/goldmark/images/transform.go", + "tpl/collections/where_test.go", + "modules/collect_test.go", + "markup/goldmark/goldmark_integration_test.go" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix menu pageRef resolution in multidimensional setups", + "revision": "d98cd4aecf25b9df78d811759ea6135b0c7610f1", + "returned": [ + "common/hugo/version_current.go", + "hugolib/site.go", + "hugolib/hugo_sites.go", + "resources/images/config.go", + ".github/workflows/aiwatchdog.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix some default site redirect woes", + "revision": "66ba63cd5f30ef8807fc524dbc67399376299bb1", + "returned": [ + "common/hugo/version_current.go", + "tpl/tplimpl/templatestore.go", + "tpl/tplimpl/templatestore_integration_test.go", + "hugolib/site.go", + ".github/workflows/aiwatchdog.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix index out of range panic in fileEventsContentPaths", + "revision": "e8577771c3275d16b7d1b31cca5e9fd3ea517416", + "returned": [ + "common/hugo/version_current.go", + "hugolib/site.go", + "hugolib/hugo_sites.go", + "resources/images/config.go", + ".github/workflows/aiwatchdog.yml" + ] + }, + { + "provider_id": "keyword-search", + "query": "Fix cascade draft panic", + "revision": "192e3c453e904cf2e79312d4335a7ee216fe659d", + "returned": [ + "hugolib/content_map_page_assembler.go", + "hugolib/content_map_test.go", + "hugolib/page__meta.go", + "config/allconfig/allconfig.go", + "config/allconfig/configlanguage.go" + ] + }, + { + "provider_id": "keyword-search", + "query": "Fix image DecodeConfig regression of WebP images from file cache", + "revision": "6ef8017f60117ad9d900cc59f10a962cd68566d6", + "returned": [ + "config/commonConfig.go", + "livereload/livereload.js", + "livereload/livereload.min.js", + "resources/image.go", + "resources/transform.go" + ] + }, + { + "provider_id": "keyword-search", + "query": "Fix index out of range panic in fileEventsContentPaths", + "revision": "e8577771c3275d16b7d1b31cca5e9fd3ea517416", + "returned": [ + "commands/server.go", + "hugolib/content_map_page_assembler.go", + "hugolib/hugo_sites_build.go", + "hugolib/page__meta.go", + "hugofs/component_fs.go" + ] + }, + { + "provider_id": "agent-default", + "query": "Fix partial decorator detection in partial with blocks with outer range break or continue", + "revision": "f42c422a12987c9ccb1927fc17da37feedd0c7fb", + "returned": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatestore.go", + "internal/warpc/js/renderkatex.bundle.js", + "hugolib/hugo_sites_build.go", + "hugolib/pages_capture.go" + ] + }, + { + "provider_id": "agent-default", + "query": "Fix some default site redirect woes", + "revision": "66ba63cd5f30ef8807fc524dbc67399376299bb1", + "returned": [ + "hugolib/site_render.go", + "hugolib/site.go", + "commands/server.go", + "config/defaultConfigProvider_test.go", + "config/commonConfig.go" + ] + }, + { + "provider_id": "graphify", + "query": "Fix partial decorator detection in partial with blocks with outer range break or continue", + "revision": "f42c422a12987c9ccb1927fc17da37feedd0c7fb", + "returned": [ + "docs/content/en/templates/shortcode.md", + "internal/warpc/js/renderkatex.bundle.js", + "docs/content/en/templates/types.md", + "hugolib/testdata/what-is-markdown.md", + "docs/content/en/functions/go-template/break.md" + ] + }, + { + "provider_id": "graphify", + "query": "Fix server errors when deleting static files or directories", + "revision": "94f3908ec6a1618361ef07c61ac0402b7d21e07a", + "returned": [ + "docs/content/en/getting-started/directory-structure.md", + "common/herrors/file_error.go", + "config/commonConfig.go", + "hugolib/filesystems/basefs.go", + "internal/js/esbuild/sourcemap.go" + ] + }, + { + "provider_id": "token-savior-regex", + "query": "Fix RenderShortcodes leaking context markers when indented", + "revision": "45e4596630b1372ab02634e4536d2b5a89452e0b", + "returned": [ + "cache/dynacache/dynacache.go", + "commands/commandeer.go", + "commands/commands.go", + "commands/config.go", + "hugolib/shortcode.go" + ] + }, + { + "provider_id": "token-savior-regex", + "query": "Fix slow server startup of very big content trees", + "revision": "555dfa207a6a631e4c8193cb6be82f54c750a8dc", + "returned": [ + "cache/filecache/filecache.go", + "cache/filecache/filecache_config_test.go", + "cache/filecache/filecache_integration_test.go", + "cache/filecache/filecache_pruner_test.go", + "cache/filecache/filecache_test.go" + ] + }, + { + "provider_id": "token-savior-regex", + "query": "Fix server rebuilds on editing content with Chinese terms", + "revision": "9e7182e9e6a21235a025c6672072b1d4e1359368", + "returned": [ + "cache/filecache/filecache.go", + "cache/filecache/filecache_config_test.go", + "cache/filecache/filecache_integration_test.go", + "cache/filecache/filecache_pruner_test.go", + "cache/filecache/filecache_test.go" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.0172, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0172, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0345, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0172, + "mean_precision_at_10": 0.0034, + "zero_hit_rate": 0.931, + "median_tokens_delivered": 31301, + "median_latency_ms": 13.503, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.0172, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0172, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0345, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0172, + "mean_precision_at_10": 0.0034, + "zero_hit_rate": 0.931, + "median_tokens_delivered": 31301, + "median_latency_ms": 13.503, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 33227, + "median_latency_ms": 14.582, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.0313, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0313, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0625, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0313, + "mean_precision_at_10": 0.0063, + "zero_hit_rate": 0.875, + "median_tokens_delivered": 29186, + "median_latency_ms": 13.308, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 20364.8, + "peak_rss_mb": 20444.3, + "delta_rss_mb": 79.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.071, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1026, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1938, + "full_recall_at_20": 0.069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0345, + "zero_hit_rate": 0.5862, + "median_tokens_delivered": 145957, + "median_latency_ms": 35.708, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.071, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1026, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1938, + "full_recall_at_20": 0.069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0345, + "zero_hit_rate": 0.5862, + "median_tokens_delivered": 145957, + "median_latency_ms": 35.708, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.0879, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.12, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2335, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0462, + "zero_hit_rate": 0.4615, + "median_tokens_delivered": 141901, + "median_latency_ms": 36.735, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.0573, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0885, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1615, + "full_recall_at_20": 0.0625, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.025, + "zero_hit_rate": 0.6875, + "median_tokens_delivered": 157334.5, + "median_latency_ms": 34.759, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 20470.3, + "peak_rss_mb": 20655.6, + "delta_rss_mb": 185.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "keyword-search", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.2541, + "full_recall_at_5": 0.069, + "mean_recall_at_10": 0.3633, + "full_recall_at_10": 0.1379, + "mean_recall_at_20": 0.5213, + "full_recall_at_20": 0.2069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1342, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.1724, + "median_tokens_delivered": 122192, + "median_latency_ms": 119.102, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.2541, + "full_recall_at_5": 0.069, + "mean_recall_at_10": 0.3633, + "full_recall_at_10": 0.1379, + "mean_recall_at_20": 0.5213, + "full_recall_at_20": 0.2069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1342, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.1724, + "median_tokens_delivered": 122192, + "median_latency_ms": 119.102, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.2592, + "full_recall_at_5": 0.0769, + "mean_recall_at_10": 0.413, + "full_recall_at_10": 0.1538, + "mean_recall_at_20": 0.6245, + "full_recall_at_20": 0.2308, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1328, + "mean_precision_at_10": 0.1308, + "zero_hit_rate": 0, + "median_tokens_delivered": 129765, + "median_latency_ms": 120.636, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.25, + "full_recall_at_5": 0.0625, + "mean_recall_at_10": 0.3229, + "full_recall_at_10": 0.125, + "mean_recall_at_20": 0.4375, + "full_recall_at_20": 0.1875, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1354, + "mean_precision_at_10": 0.075, + "zero_hit_rate": 0.3125, + "median_tokens_delivered": 113799, + "median_latency_ms": 118.363, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 20699.9, + "peak_rss_mb": 21428.8, + "delta_rss_mb": 728.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "agent-default", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.2126, + "full_recall_at_5": 0.0345, + "mean_recall_at_10": 0.3489, + "full_recall_at_10": 0.069, + "mean_recall_at_20": 0.4811, + "full_recall_at_20": 0.2069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0345, + "mean_recall_at_16000_tokens": 0.1724, + "mean_precision_at_10": 0.0931, + "zero_hit_rate": 0.2414, + "median_tokens_delivered": 88585, + "median_latency_ms": 159.224, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.2126, + "full_recall_at_5": 0.0345, + "mean_recall_at_10": 0.3489, + "full_recall_at_10": 0.069, + "mean_recall_at_20": 0.4811, + "full_recall_at_20": 0.2069, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0345, + "mean_recall_at_16000_tokens": 0.1724, + "mean_precision_at_10": 0.0931, + "zero_hit_rate": 0.2414, + "median_tokens_delivered": 88585, + "median_latency_ms": 159.224, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.4231, + "full_recall_at_5": 0.0769, + "mean_recall_at_10": 0.4899, + "full_recall_at_10": 0.0769, + "mean_recall_at_20": 0.6758, + "full_recall_at_20": 0.3077, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0769, + "mean_recall_at_16000_tokens": 0.359, + "mean_precision_at_10": 0.1462, + "zero_hit_rate": 0, + "median_tokens_delivered": 101077, + "median_latency_ms": 161.486, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.0417, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2344, + "full_recall_at_10": 0.0625, + "mean_recall_at_20": 0.3229, + "full_recall_at_20": 0.125, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0208, + "mean_precision_at_10": 0.05, + "zero_hit_rate": 0.4375, + "median_tokens_delivered": 87753, + "median_latency_ms": 155.3405, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 21446.3, + "peak_rss_mb": 22762.7, + "delta_rss_mb": 1316.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.1437, + "full_recall_at_5": 0.0345, + "mean_recall_at_10": 0.2299, + "full_recall_at_10": 0.0345, + "mean_recall_at_20": 0.3009, + "full_recall_at_20": 0.069, + "mean_recall_at_1000_tokens": 0.2348, + "mean_recall_at_4000_tokens": 0.3009, + "mean_recall_at_16000_tokens": 0.3009, + "mean_precision_at_10": 0.0552, + "zero_hit_rate": 0.3793, + "median_tokens_delivered": 1679, + "median_latency_ms": 11265.574, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.1437, + "full_recall_at_5": 0.0345, + "mean_recall_at_10": 0.2299, + "full_recall_at_10": 0.0345, + "mean_recall_at_20": 0.3009, + "full_recall_at_20": 0.069, + "mean_recall_at_1000_tokens": 0.2348, + "mean_recall_at_4000_tokens": 0.3009, + "mean_recall_at_16000_tokens": 0.3009, + "mean_precision_at_10": 0.0552, + "zero_hit_rate": 0.3793, + "median_tokens_delivered": 1679, + "median_latency_ms": 11265.574, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.1474, + "full_recall_at_5": 0.0769, + "mean_recall_at_10": 0.2628, + "full_recall_at_10": 0.0769, + "mean_recall_at_20": 0.2995, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0.2738, + "mean_recall_at_4000_tokens": 0.2995, + "mean_recall_at_16000_tokens": 0.2995, + "mean_precision_at_10": 0.0615, + "zero_hit_rate": 0.3077, + "median_tokens_delivered": 1681, + "median_latency_ms": 11288.908, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.1406, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2031, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3021, + "full_recall_at_20": 0.0625, + "mean_recall_at_1000_tokens": 0.2031, + "mean_recall_at_4000_tokens": 0.3021, + "mean_recall_at_16000_tokens": 0.3021, + "mean_precision_at_10": 0.05, + "zero_hit_rate": 0.4375, + "median_tokens_delivered": 1677.5, + "median_latency_ms": 11243.2985, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 21545, + "peak_rss_mb": 24266.8, + "delta_rss_mb": 2721.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 461910016 + } + }, + { + "provider_id": "token-savior-regex", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0718, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1236, + "full_recall_at_20": 0.0345, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1149, + "mean_recall_at_16000_tokens": 0.1236, + "mean_precision_at_10": 0.0256, + "zero_hit_rate": 0.7586, + "median_tokens_delivered": 4164, + "median_latency_ms": 6914.468, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0718, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1236, + "full_recall_at_20": 0.0345, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1149, + "mean_recall_at_16000_tokens": 0.1236, + "mean_precision_at_10": 0.0256, + "zero_hit_rate": 0.7586, + "median_tokens_delivered": 4164, + "median_latency_ms": 6914.468, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0705, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1859, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1667, + "mean_recall_at_16000_tokens": 0.1859, + "mean_precision_at_10": 0.0341, + "zero_hit_rate": 0.6154, + "median_tokens_delivered": 4180, + "median_latency_ms": 6922.142, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0729, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0729, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0729, + "mean_recall_at_16000_tokens": 0.0729, + "mean_precision_at_10": 0.0188, + "zero_hit_rate": 0.875, + "median_tokens_delivered": 4048, + "median_latency_ms": 6867.3405, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 29 + }, + "resources": { + "baseline_rss_mb": 21677.2, + "peak_rss_mb": 38296.7, + "delta_rss_mb": 16619.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "gortex", + "summary": { + "all": { + "cases": 29, + "mean_recall_at_5": 0.1063, + "full_recall_at_5": 0.0345, + "mean_recall_at_10": 0.1063, + "full_recall_at_10": 0.0345, + "mean_recall_at_20": 0.1178, + "full_recall_at_20": 0.0345, + "mean_recall_at_1000_tokens": 0.1063, + "mean_recall_at_4000_tokens": 0.1178, + "mean_recall_at_16000_tokens": 0.1178, + "mean_precision_at_10": 0.0451, + "zero_hit_rate": 0.7586, + "median_tokens_delivered": 842, + "median_latency_ms": 97576.842, + "payload_kind": [ + "tool-output" + ], + "unavailable": 12 + }, + "baseline_missed": { + "cases": 29, + "mean_recall_at_5": 0.1063, + "full_recall_at_5": 0.0345, + "mean_recall_at_10": 0.1063, + "full_recall_at_10": 0.0345, + "mean_recall_at_20": 0.1178, + "full_recall_at_20": 0.0345, + "mean_recall_at_1000_tokens": 0.1063, + "mean_recall_at_4000_tokens": 0.1178, + "mean_recall_at_16000_tokens": 0.1178, + "mean_precision_at_10": 0.0451, + "zero_hit_rate": 0.7586, + "median_tokens_delivered": 842, + "median_latency_ms": 97576.842, + "payload_kind": [ + "tool-output" + ], + "unavailable": 12 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 13, + "mean_recall_at_5": 0.1474, + "full_recall_at_5": 0.0769, + "mean_recall_at_10": 0.1474, + "full_recall_at_10": 0.0769, + "mean_recall_at_20": 0.1474, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0.1474, + "mean_recall_at_4000_tokens": 0.1474, + "mean_recall_at_16000_tokens": 0.1474, + "mean_precision_at_10": 0.066, + "zero_hit_rate": 0.6923, + "median_tokens_delivered": 1146, + "median_latency_ms": 114861.576, + "payload_kind": [ + "tool-output" + ], + "unavailable": 4 + }, + "no_path_leak": { + "cases": 16, + "mean_recall_at_5": 0.0729, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0729, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0938, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.0729, + "mean_recall_at_4000_tokens": 0.0938, + "mean_recall_at_16000_tokens": 0.0938, + "mean_precision_at_10": 0.0281, + "zero_hit_rate": 0.8125, + "median_tokens_delivered": 19.5, + "median_latency_ms": 89469.0455, + "payload_kind": [ + "tool-output" + ], + "unavailable": 8 + } + }, + "outcomes": { + "answered": 15, + "indexed-but-returned-nothing": 3, + "did-not-index": 11 + }, + "resources": { + "baseline_rss_mb": 24468.6, + "peak_rss_mb": 51326.4, + "delta_rss_mb": 26857.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23450, + "latency_ms": 28.963, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43237, + "latency_ms": 17.374, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [], + "missed": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15701, + "latency_ms": 16.107, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [], + "missed": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23223, + "latency_ms": 16.683, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [], + "missed": [ + "commands/server.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31301, + "latency_ms": 16.888, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33227, + "latency_ms": 17.03, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [ + "markup/goldmark/goldmark_integration_test.go" + ], + "missed": [ + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 31381, + "latency_ms": 15.001, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26363, + "latency_ms": 15.148, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32262, + "latency_ms": 15.604, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28830, + "latency_ms": 13.342, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18869, + "latency_ms": 13.164, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [ + "hugolib/embedded_templates_test.go" + ], + "missed": [ + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 23902, + "latency_ms": 13.274, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27186, + "latency_ms": 13.503, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33298, + "latency_ms": 13.203, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [], + "missed": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38765, + "latency_ms": 12.571, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23248, + "latency_ms": 12.88, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32875, + "latency_ms": 13.258, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33433, + "latency_ms": 12.741, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39664, + "latency_ms": 13.963, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24224, + "latency_ms": 14.146, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22664, + "latency_ms": 13.112, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27862, + "latency_ms": 12.91, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44378, + "latency_ms": 12.667, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84924, + "latency_ms": 13.156, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43205, + "latency_ms": 13.034, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43171, + "latency_ms": 14.582, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [], + "missed": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29542, + "latency_ms": 14.689, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34959, + "latency_ms": 13.029, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30727, + "latency_ms": 16.148, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 135284, + "latency_ms": 162.155, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 141901, + "latency_ms": 34.359, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [ + "hugolib/cascade_test.go" + ], + "missed": [ + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 147188, + "latency_ms": 37.458, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [], + "missed": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 167064, + "latency_ms": 35.985, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [], + "missed": [ + "commands/server.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 158300, + "latency_ms": 36.293, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/site.go" + ], + "missed": [ + "hugolib/menu_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 160960, + "latency_ms": 34.17, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [], + "missed": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 156369, + "latency_ms": 33.645, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123900, + "latency_ms": 59.738, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145957, + "latency_ms": 32.199, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 167158, + "latency_ms": 36.499, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 168902, + "latency_ms": 34.354, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 143955, + "latency_ms": 36.9, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [ + "hugolib/integrationtest_builder.go" + ], + "missed": [ + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 148660, + "latency_ms": 35.708, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/sitesmatrix/dimensions.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.2857, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 167157, + "latency_ms": 34.592, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "config/allconfig/allconfig.go", + "hugolib/site.go" + ], + "missed": [ + ".prettierignore", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 140039, + "latency_ms": 37.06, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 158300, + "latency_ms": 32.767, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145753, + "latency_ms": 33.577, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 133497, + "latency_ms": 55.361, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 141218, + "latency_ms": 36.938, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [ + "hugolib/rebuild_test.go" + ], + "missed": [ + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 167064, + "latency_ms": 34.02, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145952, + "latency_ms": 33.378, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159604, + "latency_ms": 34.458, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [ + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 133592, + "latency_ms": 36.935, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [ + "hugolib/rebuild_test.go" + ], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 135377, + "latency_ms": 51.971, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 139252, + "latency_ms": 38.731, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 133591, + "latency_ms": 36.735, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [ + "tpl/css/build_integration_test.go" + ], + "missed": [ + "resources/transform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 158549, + "latency_ms": 31.921, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 163147, + "latency_ms": 35.362, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 138542, + "latency_ms": 35.06, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [ + "tpl/tplimpl/templatetransform.go" + ], + "missed": [ + "tpl/templates/decorator_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 199275, + "latency_ms": 170.24, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 129765, + "latency_ms": 122.924, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "hugolib/cascade_test.go", + "magefile.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 128841, + "latency_ms": 63.172, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 95845, + "latency_ms": 119.102, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 142606, + "latency_ms": 160.761, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 85451, + "latency_ms": 101.258, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [ + "markup/goldmark/render_hooks.go" + ], + "missed": [ + "markup/goldmark/goldmark_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 123325, + "latency_ms": 137.527, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go" + ], + "missed": [ + "helpers/path_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 172712, + "latency_ms": 120.843, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [ + "config/commonConfig.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 105495, + "latency_ms": 99.563, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 98451, + "latency_ms": 101.516, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 155306, + "latency_ms": 160.874, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 122192, + "latency_ms": 80.41, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [ + "hugolib/segments/segments_integration_test.go" + ], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 254429, + "latency_ms": 103.848, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/sitesmatrix/dimensions.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0.2857, + "precision_at_5": 0.4, + "recall_at_10": 0.2857, + "precision_at_10": 0.2, + "recall_at_20": 0.2857, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 2, + "tokens_delivered": 115860, + "latency_ms": 115.946, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + ".prettierignore", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 98117, + "latency_ms": 83.679, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 122103, + "latency_ms": 120.193, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [ + "resources/image.go", + "resources/image_cache.go" + ], + "missed": [ + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 107986, + "latency_ms": 142.183, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 87992, + "latency_ms": 59.073, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 86071, + "latency_ms": 121.433, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [ + "tpl/tplimpl/templatestore.go" + ], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 148886, + "latency_ms": 189.505, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41091, + "latency_ms": 60.579, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 143807, + "latency_ms": 117.624, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [ + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 122785, + "latency_ms": 125.174, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [ + "commands/server.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "missed": [ + "common/paths/path_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 130099, + "latency_ms": 120.636, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go" + ], + "missed": [ + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 226391, + "latency_ms": 198.519, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "hugolib/404_test.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 88561, + "latency_ms": 91.273, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [ + "resources/transform.go" + ], + "missed": [ + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71997, + "latency_ms": 102.955, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/integrationtest_builder.go", + "hugolib/site.go" + ], + "missed": [ + "hugolib/rebuild_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 124353, + "latency_ms": 140.965, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 97547, + "latency_ms": 99.265, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 198452, + "latency_ms": 174.968, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 101077, + "latency_ms": 143.967, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "magefile.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 128397, + "latency_ms": 83.387, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 87514, + "latency_ms": 142.765, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 70031, + "latency_ms": 193.051, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 88284, + "latency_ms": 127.205, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [], + "missed": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 91001, + "latency_ms": 159.224, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [ + "commands/server.go", + "helpers/path.go" + ], + "missed": [ + "commands/commandeer.go", + "helpers/path_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 148950, + "latency_ms": 138.138, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [ + "config/commonConfig.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 63453, + "latency_ms": 119.985, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 85453, + "latency_ms": 134.179, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 96767, + "latency_ms": 182.44, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 122192, + "latency_ms": 236.292, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [ + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "missed": [ + "hugolib/integrationtest_builder.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 102590, + "latency_ms": 148.112, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/sitesmatrix/dimensions.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2857, + "precision_at_10": 0.2, + "recall_at_20": 0.2857, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 65281, + "latency_ms": 290.682, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "missed": [ + ".prettierignore", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 81226, + "latency_ms": 204.421, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [ + "hugolib/filesystems/basefs.go" + ], + "missed": [ + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 82920, + "latency_ms": 188.195, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [ + "resources/image.go", + "resources/image_cache.go" + ], + "missed": [ + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 103673, + "latency_ms": 312.203, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 87992, + "latency_ms": 84.414, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53780, + "latency_ms": 151.457, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [ + "tpl/tplimpl/templatestore.go" + ], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 117518, + "latency_ms": 222.931, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41091, + "latency_ms": 79.697, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 115596, + "latency_ms": 168.64, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [ + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 88585, + "latency_ms": 174.088, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [ + "commands/server.go", + "main_test.go" + ], + "missed": [ + "common/paths/path_test.go", + "hugolib/rebuild_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 95883, + "latency_ms": 161.486, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go" + ], + "missed": [ + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 189685, + "latency_ms": 218, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [ + "hugolib/content_map_page_assembler.go" + ], + "missed": [ + "hugolib/404_test.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 83673, + "latency_ms": 124.784, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [ + "resources/transform.go" + ], + "missed": [ + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 81033, + "latency_ms": 134.977, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/integrationtest_builder.go" + ], + "missed": [ + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 103585, + "latency_ms": 160.358, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 76525, + "latency_ms": 125.702, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [ + "tpl/templates/decorator_integration_test.go" + ], + "missed": [ + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1681, + "latency_ms": 11609.65, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 260, + "latency_ms": 11640.724, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [], + "missed": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1665, + "latency_ms": 11363.736, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [ + "hugolib/page__content.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "missed": [ + "hugolib/rendershortcodes_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 5, + "tokens_delivered": 1679, + "latency_ms": 11664.828, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1692, + "latency_ms": 12035.5, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/site.go" + ], + "missed": [ + "hugolib/menu_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 1673, + "latency_ms": 12465.414, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [ + "markup/goldmark/render_hooks.go" + ], + "missed": [ + "markup/goldmark/goldmark_integration_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1670, + "latency_ms": 11935.369, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [], + "missed": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1686, + "latency_ms": 11062.401, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [ + "config/commonConfig.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 1668, + "latency_ms": 11265.574, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1666, + "latency_ms": 11221.023, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [ + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1682, + "latency_ms": 11218.806, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3907, + "latency_ms": 11561.219, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments_integration_test.go" + ], + "missed": [ + "hugolib/segments/segments.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 5, + "tokens_delivered": 1684, + "latency_ms": 11336.249, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [ + "hugolib/content_map_page.go" + ], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.1429, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.1429, + "recall_at_4000_tokens": 0.1429, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 11, + "tokens_delivered": 1674, + "latency_ms": 11288.908, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "hugolib/site.go" + ], + "missed": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 6, + "tokens_delivered": 1660, + "latency_ms": 10943.843, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [ + "hugolib/filesystems/basefs.go" + ], + "missed": [ + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1676, + "latency_ms": 11368.487, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [ + "resources/image.go" + ], + "missed": [ + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 15, + "tokens_delivered": 1682, + "latency_ms": 11039.479, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1666, + "latency_ms": 11300.041, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2183, + "latency_ms": 11034.634, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [ + "tpl/tplimpl/templates.go" + ], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 344, + "latency_ms": 11139.544, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2365, + "latency_ms": 10906.364, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1675, + "latency_ms": 11040.773, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [ + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "missed": [ + "hugofs/component_fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1672, + "latency_ms": 11006.876, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1677, + "latency_ms": 10980.166, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [ + "internal/warpc/webp.go" + ], + "missed": [ + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 1704, + "latency_ms": 10812.77, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [ + "hugolib/404_test.go" + ], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 1685, + "latency_ms": 10852.96, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [ + "resources/transform.go" + ], + "missed": [ + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 1693, + "latency_ms": 11403.199, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/site.go" + ], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 1682, + "latency_ms": 10858.595, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1682, + "latency_ms": 11289.253, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5679, + "latency_ms": 8884.874, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4180, + "latency_ms": 6922.142, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [], + "missed": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2168, + "latency_ms": 4880.309, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [ + "hugolib/page__content.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "missed": [ + "hugolib/rendershortcodes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 8, + "tokens_delivered": 4006, + "latency_ms": 7154.064, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 5992, + "latency_ms": 8610.32, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2042, + "latency_ms": 6042.249, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [ + "markup/goldmark/goldmark_integration_test.go" + ], + "missed": [ + "markup/goldmark/render_hooks.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 5199, + "latency_ms": 13271.908, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [ + "commands/commandeer.go", + "commands/server.go" + ], + "missed": [ + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 4416, + "latency_ms": 16416.138, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2471, + "latency_ms": 6117.186, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4358, + "latency_ms": 6820.213, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6137, + "latency_ms": 8422.689, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3109, + "latency_ms": 5731.578, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4164, + "latency_ms": 6978.835, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5092, + "latency_ms": 6444.857, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [ + "config/allconfig/allconfig.go" + ], + "missed": [ + ".prettierignore", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1429, + "recall_at_20": 0.1667, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 6, + "tokens_delivered": 2187, + "latency_ms": 5971.821, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4037, + "latency_ms": 7208.738, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4680, + "latency_ms": 7596.016, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1511, + "latency_ms": 5250.498, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3947, + "latency_ms": 7325.745, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5825, + "latency_ms": 8909.98, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1767, + "latency_ms": 4937.738, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5471, + "latency_ms": 6460.193, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4059, + "latency_ms": 6779.724, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [ + "commands/server.go", + "main_test.go" + ], + "missed": [ + "common/paths/path_test.go", + "hugolib/rebuild_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 3726, + "latency_ms": 6809.358, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [ + "resources/images/images_golden_integration_test.go" + ], + "missed": [ + "internal/warpc/webp.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 13, + "tokens_delivered": 6532, + "latency_ms": 8285.951, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3210, + "latency_ms": 4867.952, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [], + "missed": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4492, + "latency_ms": 6101.771, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5477, + "latency_ms": 7388.665, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3295, + "latency_ms": 6914.468, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-09048aad7655", + "path_leak": true, + "required_files": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "found": [], + "missed": [ + "tpl/templates/decorator_integration_test.go", + "tpl/tplimpl/templatetransform.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2790, + "latency_ms": 42354.419, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-0f1c7d12000f", + "path_leak": true, + "required_files": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/hugo_sites.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2764, + "latency_ms": 49978.664, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-11f7f39913d5", + "path_leak": true, + "required_files": [ + "hugolib/cascade_test.go", + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "found": [ + "hugolib/cascade_test.go" + ], + "missed": [ + "hugolib/content_map_page_assembler.go", + "magefile.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.125, + "recall_at_20": 0.3333, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 976, + "latency_ms": 52326.482, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-161d0d475773", + "path_leak": false, + "required_files": [ + "hugolib/page__content.go", + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "found": [ + "hugolib/page__content.go" + ], + "missed": [ + "hugolib/rendershortcodes_test.go", + "markup/goldmark/hugocontext/hugocontext.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 15, + "tokens_delivered": 3680, + "latency_ms": 81361.249, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-2ffaf1fc1edb", + "path_leak": true, + "required_files": [ + "commands/server.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1358, + "latency_ms": 65110.566, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-3dff7c8c7a04", + "path_leak": true, + "required_files": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "found": [], + "missed": [ + "hugolib/menu_test.go", + "hugolib/site.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1644, + "latency_ms": 76063.987, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-479fe6c65493", + "path_leak": false, + "required_files": [ + "markup/goldmark/goldmark_integration_test.go", + "markup/goldmark/render_hooks.go" + ], + "found": [ + "markup/goldmark/render_hooks.go" + ], + "missed": [ + "markup/goldmark/goldmark_integration_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1381, + "latency_ms": 97576.842, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-7a43b928a67c", + "path_leak": true, + "required_files": [ + "commands/commandeer.go", + "commands/server.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "found": [ + "commands/server.go" + ], + "missed": [ + "commands/commandeer.go", + "helpers/path.go", + "helpers/path_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.2, + "recall_at_20": 0.25, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 1146, + "latency_ms": 114861.576, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-8a979d54d1e5", + "path_leak": false, + "required_files": [ + "config/commonConfig.go" + ], + "found": [], + "missed": [ + "config/commonConfig.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40, + "latency_ms": 45213.563, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-8b00030b344d", + "path_leak": false, + "required_files": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "found": [], + "missed": [ + "markup/goldmark/convert.go", + "markup/goldmark/toc.go", + "markup/goldmark/toc_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1254, + "latency_ms": 53377.526, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-8d6145f3c315", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "hugolib/sitesmatrix/vectorstores.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 378.89, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-8dfcece80541", + "path_leak": false, + "required_files": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "found": [], + "missed": [ + "hugolib/embedded_templates_test.go", + "tpl/templates/templates.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39, + "latency_ms": 42706.854, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-95e5e9f4ab6e", + "path_leak": true, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments.go", + "hugolib/segments/segments_integration_test.go" + ], + "found": [ + "hugolib/segments/segments.go" + ], + "missed": [ + "hugolib/integrationtest_builder.go", + "hugolib/segments/segments_integration_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 842, + "latency_ms": 120289.875, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-974772422258", + "path_leak": true, + "required_files": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "found": [], + "missed": [ + "hugofs/fileinfo.go", + "hugolib/content_map.go", + "hugolib/content_map_page.go", + "hugolib/content_map_page_assembler.go", + "hugolib/sitesmatrix/dimensions.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "resources/page/pagemeta/page_frontmatter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 300342.276, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-a77548830407", + "path_leak": true, + "required_files": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "found": [], + "missed": [ + ".prettierignore", + "config/allconfig/allconfig.go", + "hugolib/site.go", + "hugolib/site_render.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go", + "tpl/tplimpl/embedded/templates/alias.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 300274.494, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-a808f6e40d4c", + "path_leak": false, + "required_files": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "found": [], + "missed": [ + "hugolib/filesystems/basefs.go", + "hugolib/hugo_sites_build.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 300296.513, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-b5d43cdc1783", + "path_leak": true, + "required_files": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "found": [], + "missed": [ + "resources/image.go", + "resources/image_cache.go", + "resources/images/images_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 300273.632, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-b82e496c130f", + "path_leak": false, + "required_files": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 300264.837, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-c1b2e58bfb94", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/sitesmatrix/sitematrix_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 300273.322, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-c48551677c25", + "path_leak": false, + "required_files": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "found": [], + "missed": [ + "hugolib/rebuild_test.go", + "tpl/tplimpl/templates.go", + "tpl/tplimpl/templatestore.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 300340.571, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-c7b35c8704ac", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_contentnodeshifter.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 300278.398, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-ce009e3aa9fd", + "path_leak": false, + "required_files": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "found": [], + "missed": [ + "hugolib/content_map_page_assembler.go", + "hugolib/language_content_dir_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 300615.929, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-d4c0e4452c22", + "path_leak": false, + "required_files": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "found": [], + "missed": [ + "hugofs/component_fs.go", + "hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 300255.642, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-e2e64aeec57c", + "path_leak": true, + "required_files": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "found": [], + "missed": [ + "commands/server.go", + "common/paths/path_test.go", + "hugolib/rebuild_test.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 300257.757, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-e9b9b36f46cc", + "path_leak": true, + "required_files": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "found": [], + "missed": [ + "internal/warpc/webp.go", + "resources/images/images_golden_integration_test.go", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart-ff9999.webp", + "resources/images/testdata/images_golden/process/webp/crop-300x300-smart.webp" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4085, + "latency_ms": 164888.166, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-f740d7cfe0eb", + "path_leak": true, + "required_files": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "found": [], + "missed": [ + "hugolib/404_test.go", + "hugolib/content_map_page_assembler.go", + "hugolib/doctree/nodeshifttree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6578, + "latency_ms": 44277.499, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-f772998fa349", + "path_leak": false, + "required_files": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "found": [], + "missed": [ + "resources/transform.go", + "tpl/css/build_integration_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5272, + "latency_ms": 46484.899, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-f797f8490229", + "path_leak": false, + "required_files": [ + "hugolib/integrationtest_builder.go", + "hugolib/rebuild_test.go", + "hugolib/site.go" + ], + "found": [ + "hugolib/integrationtest_builder.go", + "hugolib/site.go" + ], + "missed": [ + "hugolib/rebuild_test.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.25, + "recall_at_20": 0.6667, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 1268, + "latency_ms": 58396.22, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "hugo-f8b5fa09a649", + "path_leak": false, + "required_files": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "found": [], + "missed": [ + "hugofs/decorators.go", + "hugofs/fs.go", + "hugofs/rootmapping_fs.go", + "main_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1798, + "latency_ms": 67508.58, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/V-email-manager.json b/benchmarks/context-retrieval/results/V-email-manager.json new file mode 100644 index 00000000..bd1e6f59 --- /dev/null +++ b/benchmarks/context-retrieval/results/V-email-manager.json @@ -0,0 +1,5638 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "email-manager", + "head": "6750f6ee35bd70bd5e1b31e9e5308b019e2d67d8" + }, + "tier": "small", + "tier_code_files": 102, + "corpus_counts": { + "cases": 47, + "path_leak": 21, + "baseline_blind": 26, + "multi_file": 14 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.5952, + "control": 0.0566, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "hide HF transformers import from server bundler", + "revision": "7796afb6ea63065a43e425264ab2bd72977456b8", + "returned": [ + "src/lib/auth.ts", + "tsconfig.json", + "src/lib/embeddings.ts", + "src/components/Analytics.tsx", + ".github/workflows/weekly.yml" + ] + }, + { + "provider_id": "random-files", + "query": "Fix 500 errors: add token refresh and API error handling", + "revision": "5a0df4b3fee1874f78e91f6fb283bcf3559cb10a", + "returned": [ + "src/app/page.tsx", + "src/components/EmailDetail.tsx", + "src/lib/auth.ts", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "hide HF transformers import from server bundler", + "revision": "7796afb6ea63065a43e425264ab2bd72977456b8", + "returned": [ + "package.json", + "pnpm-lock.yaml", + "src/app/page.tsx", + "next.config.ts", + "src/components/Analytics.tsx" + ] + }, + { + "provider_id": "churn-ranked", + "query": "create better-auth migrations + schema mapping for D1", + "revision": "3cbd63102c967b7ad151f44592049b080c876d6a", + "returned": [ + "package.json", + "pnpm-lock.yaml", + "src/app/page.tsx", + "next.config.ts", + "src/components/Analytics.tsx" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "migrate deploy from CF Pages to CF Workers, simplify next.config", + "revision": "2c1993e3e71ae79b67cb8b3ca68a62747c88e052", + "returned": [ + "eslint.config.js", + "next.config.ts", + "open-next.config.ts", + ".github/workflows/deploy.yml", + "src/components/saasmaker-feedback.tsx" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "Fix 500 errors: add token refresh and API error handling", + "revision": "5a0df4b3fee1874f78e91f6fb283bcf3559cb10a", + "returned": [ + "src/components/Subscriptions.tsx", + "src/app/page.tsx", + "src/components/EmailDetail.tsx", + "src/lib/gmail.ts", + "src/app/api/auth/[...nextauth]/route.ts" + ] + }, + { + "provider_id": "graphify", + "query": "migrate deploy from CF Pages to CF Workers, simplify next.config", + "revision": "2c1993e3e71ae79b67cb8b3ca68a62747c88e052", + "returned": [ + "postcss.config.mjs", + "package.json", + "next.config.ts", + "AUDIT.md", + "postcss-load-config" + ] + }, + { + "provider_id": "graphify", + "query": "create better-auth migrations + schema mapping for D1", + "revision": "3cbd63102c967b7ad151f44592049b080c876d6a", + "returned": [ + "src/lib/auth.ts", + "package.json", + "src/lib/get-access-token.ts", + "src/app/api/auth/[...all", + "src/app/api/emails/[id" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 47, + "mean_recall_at_5": 0.08, + "full_recall_at_5": 0.0426, + "mean_recall_at_10": 0.1996, + "full_recall_at_10": 0.1489, + "mean_recall_at_20": 0.3246, + "full_recall_at_20": 0.2128, + "mean_recall_at_1000_tokens": 0.0213, + "mean_recall_at_4000_tokens": 0.0566, + "mean_recall_at_16000_tokens": 0.1748, + "mean_precision_at_10": 0.034, + "zero_hit_rate": 0.5319, + "median_tokens_delivered": 16304, + "median_latency_ms": 11.997, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 40, + "mean_recall_at_5": 0.044, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0596, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2065, + "full_recall_at_20": 0.075, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0165, + "mean_recall_at_16000_tokens": 0.1304, + "mean_precision_at_10": 0.0225, + "zero_hit_rate": 0.625, + "median_tokens_delivered": 16080, + "median_latency_ms": 12.1035, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 7, + "mean_recall_at_5": 0.2857, + "full_recall_at_5": 0.2857, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.1429, + "mean_recall_at_4000_tokens": 0.2857, + "mean_recall_at_16000_tokens": 0.4286, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 76352, + "median_latency_ms": 11.223, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.0694, + "full_recall_at_5": 0.0476, + "mean_recall_at_10": 0.2123, + "full_recall_at_10": 0.1905, + "mean_recall_at_20": 0.3988, + "full_recall_at_20": 0.2857, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0218, + "mean_recall_at_16000_tokens": 0.1806, + "mean_precision_at_10": 0.0286, + "zero_hit_rate": 0.4286, + "median_tokens_delivered": 18158, + "median_latency_ms": 12.181, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 26, + "mean_recall_at_5": 0.0885, + "full_recall_at_5": 0.0385, + "mean_recall_at_10": 0.1894, + "full_recall_at_10": 0.1154, + "mean_recall_at_20": 0.2647, + "full_recall_at_20": 0.1538, + "mean_recall_at_1000_tokens": 0.0385, + "mean_recall_at_4000_tokens": 0.0846, + "mean_recall_at_16000_tokens": 0.1702, + "mean_precision_at_10": 0.0385, + "zero_hit_rate": 0.6154, + "median_tokens_delivered": 14071.5, + "median_latency_ms": 11.858, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 47 + }, + "resources": { + "baseline_rss_mb": 27068.7, + "peak_rss_mb": 27154.4, + "delta_rss_mb": 85.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 47, + "mean_recall_at_5": 0.2473, + "full_recall_at_5": 0.2128, + "mean_recall_at_10": 0.3727, + "full_recall_at_10": 0.2979, + "mean_recall_at_20": 0.5906, + "full_recall_at_20": 0.5106, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0319, + "mean_recall_at_16000_tokens": 0.2, + "mean_precision_at_10": 0.0553, + "zero_hit_rate": 0.3404, + "median_tokens_delivered": 84238, + "median_latency_ms": 18.105, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 40, + "mean_recall_at_5": 0.2406, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.3379, + "full_recall_at_10": 0.25, + "mean_recall_at_20": 0.569, + "full_recall_at_20": 0.475, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0375, + "mean_recall_at_16000_tokens": 0.185, + "mean_precision_at_10": 0.055, + "zero_hit_rate": 0.35, + "median_tokens_delivered": 88809, + "median_latency_ms": 18.2295, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 7, + "mean_recall_at_5": 0.2857, + "full_recall_at_5": 0.2857, + "mean_recall_at_10": 0.5714, + "full_recall_at_10": 0.5714, + "mean_recall_at_20": 0.7143, + "full_recall_at_20": 0.7143, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.2857, + "mean_precision_at_10": 0.0571, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 81053, + "median_latency_ms": 17.401, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.125, + "full_recall_at_5": 0.0952, + "mean_recall_at_10": 0.2817, + "full_recall_at_10": 0.1905, + "mean_recall_at_20": 0.4861, + "full_recall_at_20": 0.381, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0238, + "mean_recall_at_16000_tokens": 0.119, + "mean_precision_at_10": 0.0524, + "zero_hit_rate": 0.4286, + "median_tokens_delivered": 86151, + "median_latency_ms": 18.636, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 26, + "mean_recall_at_5": 0.3462, + "full_recall_at_5": 0.3077, + "mean_recall_at_10": 0.4462, + "full_recall_at_10": 0.3846, + "mean_recall_at_20": 0.675, + "full_recall_at_20": 0.6154, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0385, + "mean_recall_at_16000_tokens": 0.2654, + "mean_precision_at_10": 0.0577, + "zero_hit_rate": 0.2692, + "median_tokens_delivered": 83901, + "median_latency_ms": 17.8015, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 47 + }, + "resources": { + "baseline_rss_mb": 27171.1, + "peak_rss_mb": 27284.6, + "delta_rss_mb": 113.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 47, + "mean_recall_at_5": 0.4413, + "full_recall_at_5": 0.383, + "mean_recall_at_10": 0.5473, + "full_recall_at_10": 0.4681, + "mean_recall_at_20": 0.5952, + "full_recall_at_20": 0.5106, + "mean_recall_at_1000_tokens": 0.5952, + "mean_recall_at_4000_tokens": 0.5952, + "mean_recall_at_16000_tokens": 0.5952, + "mean_precision_at_10": 0.0996, + "zero_hit_rate": 0.2766, + "median_tokens_delivered": 500, + "median_latency_ms": 119.982, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 40, + "mean_recall_at_5": 0.4185, + "full_recall_at_5": 0.35, + "mean_recall_at_10": 0.4931, + "full_recall_at_10": 0.4, + "mean_recall_at_20": 0.5494, + "full_recall_at_20": 0.45, + "mean_recall_at_1000_tokens": 0.5494, + "mean_recall_at_4000_tokens": 0.5494, + "mean_recall_at_16000_tokens": 0.5494, + "mean_precision_at_10": 0.0929, + "zero_hit_rate": 0.3, + "median_tokens_delivered": 504.5, + "median_latency_ms": 133.504, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 7, + "mean_recall_at_5": 0.5714, + "full_recall_at_5": 0.5714, + "mean_recall_at_10": 0.8571, + "full_recall_at_10": 0.8571, + "mean_recall_at_20": 0.8571, + "full_recall_at_20": 0.8571, + "mean_recall_at_1000_tokens": 0.8571, + "mean_recall_at_4000_tokens": 0.8571, + "mean_recall_at_16000_tokens": 0.8571, + "mean_precision_at_10": 0.1383, + "zero_hit_rate": 0.1429, + "median_tokens_delivered": 477, + "median_latency_ms": 104.073, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.6567, + "full_recall_at_5": 0.5714, + "mean_recall_at_10": 0.7321, + "full_recall_at_10": 0.619, + "mean_recall_at_20": 0.7798, + "full_recall_at_20": 0.6667, + "mean_recall_at_1000_tokens": 0.7798, + "mean_recall_at_4000_tokens": 0.7798, + "mean_recall_at_16000_tokens": 0.7798, + "mean_precision_at_10": 0.1303, + "zero_hit_rate": 0, + "median_tokens_delivered": 468, + "median_latency_ms": 134.72, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 26, + "mean_recall_at_5": 0.2673, + "full_recall_at_5": 0.2308, + "mean_recall_at_10": 0.3981, + "full_recall_at_10": 0.3462, + "mean_recall_at_20": 0.4462, + "full_recall_at_20": 0.3846, + "mean_recall_at_1000_tokens": 0.4462, + "mean_recall_at_4000_tokens": 0.4462, + "mean_recall_at_16000_tokens": 0.4462, + "mean_precision_at_10": 0.0749, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 547.5, + "median_latency_ms": 116.062, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 47 + }, + "resources": { + "baseline_rss_mb": 27263.7, + "peak_rss_mb": 28594.8, + "delta_rss_mb": 1331, + "scope": "system-wide-upper-bound", + "index_bytes_added": 89001984 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 47, + "mean_recall_at_5": 0.395, + "full_recall_at_5": 0.3404, + "mean_recall_at_10": 0.4473, + "full_recall_at_10": 0.383, + "mean_recall_at_20": 0.505, + "full_recall_at_20": 0.4255, + "mean_recall_at_1000_tokens": 0.3766, + "mean_recall_at_4000_tokens": 0.5023, + "mean_recall_at_16000_tokens": 0.505, + "mean_precision_at_10": 0.1449, + "zero_hit_rate": 0.383, + "median_tokens_delivered": 1677, + "median_latency_ms": 915.738, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 40, + "mean_recall_at_5": 0.3892, + "full_recall_at_5": 0.325, + "mean_recall_at_10": 0.4506, + "full_recall_at_10": 0.375, + "mean_recall_at_20": 0.4933, + "full_recall_at_20": 0.4, + "mean_recall_at_1000_tokens": 0.3675, + "mean_recall_at_4000_tokens": 0.4902, + "mean_recall_at_16000_tokens": 0.4933, + "mean_precision_at_10": 0.159, + "zero_hit_rate": 0.375, + "median_tokens_delivered": 1669.5, + "median_latency_ms": 967.8655, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 7, + "mean_recall_at_5": 0.4286, + "full_recall_at_5": 0.4286, + "mean_recall_at_10": 0.4286, + "full_recall_at_10": 0.4286, + "mean_recall_at_20": 0.5714, + "full_recall_at_20": 0.5714, + "mean_recall_at_1000_tokens": 0.4286, + "mean_recall_at_4000_tokens": 0.5714, + "mean_recall_at_16000_tokens": 0.5714, + "mean_precision_at_10": 0.0643, + "zero_hit_rate": 0.4286, + "median_tokens_delivered": 1685, + "median_latency_ms": 909.323, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.5734, + "full_recall_at_5": 0.5238, + "mean_recall_at_10": 0.5734, + "full_recall_at_10": 0.5238, + "mean_recall_at_20": 0.5913, + "full_recall_at_20": 0.5238, + "mean_recall_at_1000_tokens": 0.5317, + "mean_recall_at_4000_tokens": 0.5913, + "mean_recall_at_16000_tokens": 0.5913, + "mean_precision_at_10": 0.1667, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 1679, + "median_latency_ms": 915.738, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 26, + "mean_recall_at_5": 0.251, + "full_recall_at_5": 0.1923, + "mean_recall_at_10": 0.3455, + "full_recall_at_10": 0.2692, + "mean_recall_at_20": 0.4353, + "full_recall_at_20": 0.3462, + "mean_recall_at_1000_tokens": 0.2513, + "mean_recall_at_4000_tokens": 0.4305, + "mean_recall_at_16000_tokens": 0.4353, + "mean_precision_at_10": 0.1273, + "zero_hit_rate": 0.4615, + "median_tokens_delivered": 1547.5, + "median_latency_ms": 921.3515, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 46, + "indexed-but-returned-nothing": 1 + }, + "resources": { + "baseline_rss_mb": 28038.8, + "peak_rss_mb": 28757.2, + "delta_rss_mb": 718.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 16347136 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "email-manager-0476e5ea7a56", + "path_leak": true, + "required_files": [ + "src/app/api/auth/[...all]/route.ts" + ], + "found": [], + "missed": [ + "src/app/api/auth/[...all]/route.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82897, + "latency_ms": 25.638, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-0d87af9a34b0", + "path_leak": true, + "required_files": [ + "src/components/GmailFilterBuilder.tsx" + ], + "found": [ + "src/components/GmailFilterBuilder.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 18158, + "latency_ms": 11.718, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-0eddf5a467de", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16175, + "latency_ms": 16.613, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-13cc40c36dae", + "path_leak": false, + "required_files": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "found": [ + "src/lib/embeddings.ts" + ], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 80783, + "latency_ms": 14.107, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-16cd9b84e276", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "next.config.ts", + "wrangler.toml" + ], + "found": [ + "next.config.ts", + "wrangler.toml" + ], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 6886, + "latency_ms": 13.173, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-16da9a4b0f8e", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 43583, + "latency_ms": 12.952, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-1f4a2fcb88d5", + "path_leak": false, + "required_files": [ + "landing-astro/src/pages/index.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/pages/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21416, + "latency_ms": 12.527, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-1fc66e44e95f", + "path_leak": true, + "required_files": [ + "src/app/opengraph-image.tsx" + ], + "found": [], + "missed": [ + "src/app/opengraph-image.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12610, + "latency_ms": 12.043, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-2961ceed2611", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx" + ], + "found": [ + "src/app/app/HomeClient.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 17, + "tokens_delivered": 15243, + "latency_ms": 11.234, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-2aedecaad649", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7796, + "latency_ms": 12.346, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-3338db62bd1b", + "path_leak": false, + "required_files": [ + "landing-astro/src/layouts/Layout.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/layouts/Layout.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18126, + "latency_ms": 11.732, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-33790a7fd4e5", + "path_leak": true, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 14, + "tokens_delivered": 9805, + "latency_ms": 12.376, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-338b73db29bb", + "path_leak": true, + "required_files": [ + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15985, + "latency_ms": 12.457, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-35501ddbd7e8", + "path_leak": true, + "required_files": [ + "scripts/cf-build.mjs" + ], + "found": [ + "scripts/cf-build.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 77583, + "latency_ms": 11.223, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-35efc8549441", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "found": [ + "src/components/posthog-provider.tsx", + "src/pages/HomeClient.tsx" + ], + "missed": [ + "public/_headers", + "src/lib/analytics.ts", + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "tests/open-app.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.125, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.125, + "first_hit_rank": 6, + "tokens_delivered": 51001, + "latency_ms": 11.685, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-3c474e85a025", + "path_leak": true, + "required_files": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "found": [ + "landing-astro/src/pages/faq.astro", + "src/pages/PrivacyPage.tsx" + ], + "missed": [ + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 12547, + "latency_ms": 11.222, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-3cbd63102c96", + "path_leak": true, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8123, + "latency_ms": 11.437, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-45ab8efac77e", + "path_leak": false, + "required_files": [ + "src/components/EmailDetail.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailDetail.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12152, + "latency_ms": 11.903, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-4f284d35f52f", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10212, + "latency_ms": 12.182, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-50270cf8bf36", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailList.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105836, + "latency_ms": 12.181, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-5dd4caedcd8d", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 84224, + "latency_ms": 10.807, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-5e56f1efdc60", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "tsconfig.json" + ], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 78003, + "latency_ms": 12.499, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-61a651298526", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [], + "missed": [ + "src/components/EmailList.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105836, + "latency_ms": 14.504, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-63eecd0ec045", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "found": [ + "worker.mjs" + ], + "missed": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 110079, + "latency_ms": 12.208, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-64def25acbb3", + "path_leak": true, + "required_files": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "src/lib/auth.ts", + "wrangler.toml" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 79248, + "latency_ms": 14.84, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-6acde8cf1b54", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12295, + "latency_ms": 12.85, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-6cf000313340", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8308, + "latency_ms": 11.813, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-7796afb6ea63", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [ + "wrangler.toml" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 82790, + "latency_ms": 10.724, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-7b0fe5379d1a", + "path_leak": false, + "required_files": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "found": [ + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0.8, + "precision_at_5": 0.8, + "recall_at_10": 0.8, + "precision_at_10": 0.4, + "recall_at_20": 0.8, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.8, + "first_hit_rank": 1, + "tokens_delivered": 40120, + "latency_ms": 11.275, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-88e6e8039b06", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17703, + "latency_ms": 11.985, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-8a3b6110595c", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [], + "missed": [ + "src/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8549, + "latency_ms": 11.788, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-92f5176080c2", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3864, + "latency_ms": 12.447, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-95a35f10d8b7", + "path_leak": true, + "required_files": [ + "src/worker.ts", + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "found": [ + "wrangler.toml" + ], + "missed": [ + "src/worker.ts", + "tests/open-app.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 72616, + "latency_ms": 11.369, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-ade59255e434", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 129236, + "latency_ms": 11.651, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-af92947d28e7", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5563, + "latency_ms": 10.639, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-b3822a197c25", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12349, + "latency_ms": 12.991, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-b5e31e012879", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6013, + "latency_ms": 11.464, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-ba9e6497f9eb", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7381, + "latency_ms": 11.997, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-c64f6290bdd8", + "path_leak": true, + "required_files": [ + "wrangler.toml" + ], + "found": [ + "wrangler.toml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 76352, + "latency_ms": 10.93, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-c7d7703e1f35", + "path_leak": false, + "required_files": [ + "src/app/page.tsx" + ], + "found": [ + "src/app/page.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 6707, + "latency_ms": 11.651, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-cd131b24cb29", + "path_leak": true, + "required_files": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16304, + "latency_ms": 12.937, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-d1746f53d55e", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "found": [], + "missed": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 75981, + "latency_ms": 11.746, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-e1bbe3e69163", + "path_leak": false, + "required_files": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "found": [ + "src/lib/security-headers.ts" + ], + "missed": [ + "src/worker.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 74523, + "latency_ms": 12.412, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-e52fcd58cd2d", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 42852, + "latency_ms": 11.294, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-edc4fd5d594b", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 9715, + "latency_ms": 13.969, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-f1c2f027bafe", + "path_leak": false, + "required_files": [ + "tests/example.spec.ts" + ], + "found": [], + "missed": [ + "tests/example.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12900, + "latency_ms": 11.43, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "email-manager-f24464fd6750", + "path_leak": true, + "required_files": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "found": [ + "src/components/TriageQueues.tsx" + ], + "missed": [ + "src/components/EmailDetail.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "recall_at_5": 0.125, + "precision_at_5": 0.2, + "recall_at_10": 0.125, + "precision_at_10": 0.1, + "recall_at_20": 0.125, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.125, + "recall_at_16000_tokens": 0.125, + "first_hit_rank": 3, + "tokens_delivered": 14806, + "latency_ms": 12.164, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-0476e5ea7a56", + "path_leak": true, + "required_files": [ + "src/app/api/auth/[...all]/route.ts" + ], + "found": [], + "missed": [ + "src/app/api/auth/[...all]/route.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86151, + "latency_ms": 21.89, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-0d87af9a34b0", + "path_leak": true, + "required_files": [ + "src/components/GmailFilterBuilder.tsx" + ], + "found": [], + "missed": [ + "src/components/GmailFilterBuilder.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 114507, + "latency_ms": 18.681, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-0eddf5a467de", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 92915, + "latency_ms": 19.419, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-13cc40c36dae", + "path_leak": false, + "required_files": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "found": [ + "next.config.ts" + ], + "missed": [ + "src/lib/embeddings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 81053, + "latency_ms": 19.615, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-16cd9b84e276", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "next.config.ts", + "wrangler.toml" + ], + "found": [ + "next.config.ts", + "wrangler.toml" + ], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 78112, + "latency_ms": 18.799, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-16da9a4b0f8e", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 43583, + "latency_ms": 18.105, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-1f4a2fcb88d5", + "path_leak": false, + "required_files": [ + "landing-astro/src/pages/index.astro" + ], + "found": [ + "landing-astro/src/pages/index.astro" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 136353, + "latency_ms": 25.729, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-1fc66e44e95f", + "path_leak": true, + "required_files": [ + "src/app/opengraph-image.tsx" + ], + "found": [], + "missed": [ + "src/app/opengraph-image.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 88809, + "latency_ms": 16.753, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-2961ceed2611", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx" + ], + "found": [], + "missed": [ + "src/app/app/HomeClient.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 114539, + "latency_ms": 17.4, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-2aedecaad649", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [ + "wrangler.toml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 83883, + "latency_ms": 16.67, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-3338db62bd1b", + "path_leak": false, + "required_files": [ + "landing-astro/src/layouts/Layout.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/layouts/Layout.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 136353, + "latency_ms": 22.644, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-33790a7fd4e5", + "path_leak": true, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json", + "tsconfig.json" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 46302, + "latency_ms": 17.763, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-338b73db29bb", + "path_leak": true, + "required_files": [ + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 139417, + "latency_ms": 22.716, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-35501ddbd7e8", + "path_leak": true, + "required_files": [ + "scripts/cf-build.mjs" + ], + "found": [], + "missed": [ + "scripts/cf-build.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 77940, + "latency_ms": 19.54, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-35efc8549441", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "found": [ + "src/lib/foundry-monitoring.ts", + "src/pages/HomeClient.tsx" + ], + "missed": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "tests/open-app.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 90883, + "latency_ms": 20.738, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-3c474e85a025", + "path_leak": true, + "required_files": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "found": [], + "missed": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 135232, + "latency_ms": 20.234, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-3cbd63102c96", + "path_leak": true, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 83842, + "latency_ms": 16.998, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-45ab8efac77e", + "path_leak": false, + "required_files": [ + "src/components/EmailDetail.tsx" + ], + "found": [ + "src/components/EmailDetail.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 114463, + "latency_ms": 17.596, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-4f284d35f52f", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 46156, + "latency_ms": 17.27, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-50270cf8bf36", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [ + "src/components/EmailList.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 114463, + "latency_ms": 18.636, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-5dd4caedcd8d", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 84238, + "latency_ms": 15.942, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-5e56f1efdc60", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json", + "tsconfig.json" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 80214, + "latency_ms": 15.451, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-61a651298526", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [ + "src/components/EmailList.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 114463, + "latency_ms": 17.072, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-63eecd0ec045", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "found": [], + "missed": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 113173, + "latency_ms": 18.341, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-64def25acbb3", + "path_leak": true, + "required_files": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "src/lib/auth.ts", + "wrangler.toml" + ], + "found": [ + "src/lib/auth.ts", + "wrangler.toml" + ], + "missed": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 83842, + "latency_ms": 18.118, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-6acde8cf1b54", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [ + "next.config.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 78526, + "latency_ms": 17.553, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-6cf000313340", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 83868, + "latency_ms": 16.767, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-7796afb6ea63", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [ + "wrangler.toml" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 81053, + "latency_ms": 16.664, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-7b0fe5379d1a", + "path_leak": false, + "required_files": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "found": [ + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.8, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 6, + "tokens_delivered": 40120, + "latency_ms": 19.862, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-88e6e8039b06", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 139417, + "latency_ms": 22.316, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-8a3b6110595c", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 83901, + "latency_ms": 17.823, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-92f5176080c2", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 83844, + "latency_ms": 17.401, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-95a35f10d8b7", + "path_leak": true, + "required_files": [ + "src/worker.ts", + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "found": [ + "src/worker.ts", + "wrangler.toml" + ], + "missed": [ + "tests/open-app.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 90651, + "latency_ms": 17.929, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-ade59255e434", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 141342, + "latency_ms": 19.394, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-af92947d28e7", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 83901, + "latency_ms": 15.978, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-b3822a197c25", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [ + "next.config.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88809, + "latency_ms": 15.818, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-b5e31e012879", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 81051, + "latency_ms": 15.282, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-ba9e6497f9eb", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [ + "next.config.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 77903, + "latency_ms": 15.66, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-c64f6290bdd8", + "path_leak": true, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 78303, + "latency_ms": 15.723, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-c7d7703e1f35", + "path_leak": false, + "required_files": [ + "src/app/page.tsx" + ], + "found": [ + "src/app/page.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 83818, + "latency_ms": 18.885, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-cd131b24cb29", + "path_leak": true, + "required_files": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140021, + "latency_ms": 19.788, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-d1746f53d55e", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "found": [], + "missed": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 92912, + "latency_ms": 18.073, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-e1bbe3e69163", + "path_leak": false, + "required_files": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "found": [], + "missed": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 88536, + "latency_ms": 17.78, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-e52fcd58cd2d", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 42852, + "latency_ms": 19.794, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-edc4fd5d594b", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 78587, + "latency_ms": 19.409, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-f1c2f027bafe", + "path_leak": false, + "required_files": [ + "tests/example.spec.ts" + ], + "found": [], + "missed": [ + "tests/example.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 113080, + "latency_ms": 19.584, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "email-manager-f24464fd6750", + "path_leak": true, + "required_files": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "found": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/worker.ts" + ], + "missed": [ + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css" + ], + "recall_at_5": 0.125, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.2, + "recall_at_20": 0.375, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 91399, + "latency_ms": 18.946, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-0476e5ea7a56", + "path_leak": true, + "required_files": [ + "src/app/api/auth/[...all]/route.ts" + ], + "found": [ + "src/app/api/auth/[...all]/route.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 370, + "latency_ms": 210.391, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-0d87af9a34b0", + "path_leak": true, + "required_files": [ + "src/components/GmailFilterBuilder.tsx" + ], + "found": [ + "src/components/GmailFilterBuilder.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 315, + "latency_ms": 134.72, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-0eddf5a467de", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 717, + "latency_ms": 151.995, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-13cc40c36dae", + "path_leak": false, + "required_files": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "found": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 666, + "latency_ms": 101.509, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-16cd9b84e276", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "next.config.ts", + "wrangler.toml" + ], + "found": [ + "next.config.ts" + ], + "missed": [ + ".husky/pre-push", + "wrangler.toml" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 479, + "latency_ms": 119.982, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-16da9a4b0f8e", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 314, + "latency_ms": 74.013, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-1f4a2fcb88d5", + "path_leak": false, + "required_files": [ + "landing-astro/src/pages/index.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/pages/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51, + "latency_ms": 184.161, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-1fc66e44e95f", + "path_leak": true, + "required_files": [ + "src/app/opengraph-image.tsx" + ], + "found": [ + "src/app/opengraph-image.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 509, + "latency_ms": 168.695, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-2961ceed2611", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx" + ], + "found": [ + "src/app/app/HomeClient.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 286, + "latency_ms": 131.553, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-2aedecaad649", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 554, + "latency_ms": 119.355, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-3338db62bd1b", + "path_leak": false, + "required_files": [ + "landing-astro/src/layouts/Layout.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/layouts/Layout.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 786, + "latency_ms": 200.093, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-33790a7fd4e5", + "path_leak": true, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "tsconfig.json" + ], + "missed": [ + "package.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 526, + "latency_ms": 96.227, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-338b73db29bb", + "path_leak": true, + "required_files": [ + "scripts/verify-public-indexing.mjs" + ], + "found": [ + "scripts/verify-public-indexing.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 251, + "latency_ms": 184.289, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-35501ddbd7e8", + "path_leak": true, + "required_files": [ + "scripts/cf-build.mjs" + ], + "found": [ + "scripts/cf-build.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 597, + "latency_ms": 150.99, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-35efc8549441", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "found": [ + "src/components/posthog-provider.tsx", + "src/lib/foundry-monitoring.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "missed": [ + "public/_headers", + "src/lib/analytics.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.4, + "recall_at_10": 0.25, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 649, + "latency_ms": 152.042, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-3c474e85a025", + "path_leak": true, + "required_files": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "found": [ + "src/pages/PrivacyPage.tsx" + ], + "missed": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 8, + "tokens_delivered": 272, + "latency_ms": 221.529, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-3cbd63102c96", + "path_leak": true, + "required_files": [ + ".husky/pre-push" + ], + "found": [ + ".husky/pre-push" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 251, + "latency_ms": 95.617, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-45ab8efac77e", + "path_leak": false, + "required_files": [ + "src/components/EmailDetail.tsx" + ], + "found": [ + "src/components/EmailDetail.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 396, + "latency_ms": 145.215, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-4f284d35f52f", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 611, + "latency_ms": 82.553, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-50270cf8bf36", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [ + "src/components/EmailList.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 442, + "latency_ms": 16.86, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-5dd4caedcd8d", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 455, + "latency_ms": 111.299, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-5e56f1efdc60", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 654, + "latency_ms": 96.432, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-61a651298526", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [ + "src/components/EmailList.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 442, + "latency_ms": 19.938, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-63eecd0ec045", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "found": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 14, + "tokens_delivered": 547, + "latency_ms": 132.288, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-64def25acbb3", + "path_leak": true, + "required_files": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "src/lib/auth.ts", + "wrangler.toml" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "wrangler.toml" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1429, + "recall_at_20": 0.25, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 468, + "latency_ms": 97.553, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-6acde8cf1b54", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 630, + "latency_ms": 101.87, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-6cf000313340", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 637, + "latency_ms": 108.329, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-7796afb6ea63", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 584, + "latency_ms": 97.066, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-7b0fe5379d1a", + "path_leak": false, + "required_files": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "found": [ + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx" + ], + "missed": [ + ".gitignore", + "src/lib/auth.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.6, + "precision_at_10": 0.4286, + "recall_at_20": 0.6, + "precision_at_20": 0.4286, + "recall_at_1000_tokens": 0.6, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 2, + "tokens_delivered": 426, + "latency_ms": 89.855, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-88e6e8039b06", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22, + "latency_ms": 199.19, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-8a3b6110595c", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 621, + "latency_ms": 96.836, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-92f5176080c2", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 477, + "latency_ms": 104.073, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-95a35f10d8b7", + "path_leak": true, + "required_files": [ + "src/worker.ts", + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "found": [ + "src/worker.ts" + ], + "missed": [ + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 953, + "latency_ms": 148.7, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-ade59255e434", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22, + "latency_ms": 225.306, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-af92947d28e7", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33, + "latency_ms": 96.68, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-b3822a197c25", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [ + "next.config.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 583, + "latency_ms": 110.169, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-b5e31e012879", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 60, + "latency_ms": 98.151, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-ba9e6497f9eb", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 438, + "latency_ms": 92.589, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-c64f6290bdd8", + "path_leak": true, + "required_files": [ + "wrangler.toml" + ], + "found": [ + "wrangler.toml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 7, + "latency_ms": 90.626, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-c7d7703e1f35", + "path_leak": false, + "required_files": [ + "src/app/page.tsx" + ], + "found": [ + "src/app/page.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 548, + "latency_ms": 112.769, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-cd131b24cb29", + "path_leak": true, + "required_files": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "found": [ + "landing-astro/src/layouts/Layout.astro" + ], + "missed": [ + "index.html", + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 8, + "tokens_delivered": 714, + "latency_ms": 198.425, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-d1746f53d55e", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "found": [], + "missed": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 599, + "latency_ms": 190.046, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-e1bbe3e69163", + "path_leak": false, + "required_files": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "found": [ + "src/worker.ts" + ], + "missed": [ + "src/lib/security-headers.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 744, + "latency_ms": 148.468, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-e52fcd58cd2d", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 610, + "latency_ms": 162.518, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-edc4fd5d594b", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 500, + "latency_ms": 223.122, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-f1c2f027bafe", + "path_leak": false, + "required_files": [ + "tests/example.spec.ts" + ], + "found": [ + "tests/example.spec.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 273, + "latency_ms": 196.14, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "email-manager-f24464fd6750", + "path_leak": true, + "required_files": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "found": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx" + ], + "missed": [ + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "recall_at_5": 0.375, + "precision_at_5": 0.6, + "recall_at_10": 0.375, + "precision_at_10": 0.3, + "recall_at_20": 0.375, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.375, + "recall_at_4000_tokens": 0.375, + "recall_at_16000_tokens": 0.375, + "first_hit_rank": 3, + "tokens_delivered": 582, + "latency_ms": 300.551, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-0476e5ea7a56", + "path_leak": true, + "required_files": [ + "src/app/api/auth/[...all]/route.ts" + ], + "found": [], + "missed": [ + "src/app/api/auth/[...all]/route.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5213, + "latency_ms": 1274.251, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-0d87af9a34b0", + "path_leak": true, + "required_files": [ + "src/components/GmailFilterBuilder.tsx" + ], + "found": [ + "src/components/GmailFilterBuilder.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 7378, + "latency_ms": 984.672, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-0eddf5a467de", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [], + "missed": [ + "src/lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2347, + "latency_ms": 1769.723, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-13cc40c36dae", + "path_leak": false, + "required_files": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "found": [], + "missed": [ + "next.config.ts", + "src/lib/embeddings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 157, + "latency_ms": 884.276, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-16cd9b84e276", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "next.config.ts", + "wrangler.toml" + ], + "found": [ + "next.config.ts" + ], + "missed": [ + ".husky/pre-push", + "wrangler.toml" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 564, + "latency_ms": 753.882, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-16da9a4b0f8e", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4690, + "latency_ms": 722.37, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-1f4a2fcb88d5", + "path_leak": false, + "required_files": [ + "landing-astro/src/pages/index.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/pages/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 438, + "latency_ms": 1575.058, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-1fc66e44e95f", + "path_leak": true, + "required_files": [ + "src/app/opengraph-image.tsx" + ], + "found": [ + "src/app/opengraph-image.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 767, + "latency_ms": 951.059, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-2961ceed2611", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx" + ], + "found": [ + "src/app/app/HomeClient.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 6017, + "latency_ms": 1280.27, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-2aedecaad649", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6397, + "latency_ms": 900.843, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-3338db62bd1b", + "path_leak": false, + "required_files": [ + "landing-astro/src/layouts/Layout.astro" + ], + "found": [], + "missed": [ + "landing-astro/src/layouts/Layout.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1023, + "latency_ms": 1754.545, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-33790a7fd4e5", + "path_leak": true, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json", + "tsconfig.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 418, + "latency_ms": 835.116, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-338b73db29bb", + "path_leak": true, + "required_files": [ + "scripts/verify-public-indexing.mjs" + ], + "found": [ + "scripts/verify-public-indexing.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1544, + "latency_ms": 2232.136, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-35501ddbd7e8", + "path_leak": true, + "required_files": [ + "scripts/cf-build.mjs" + ], + "found": [], + "missed": [ + "scripts/cf-build.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 286, + "latency_ms": 909.323, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-35efc8549441", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "found": [ + "src/lib/foundry-monitoring.ts", + "src/lib/security-headers.ts" + ], + "missed": [ + "public/_headers", + "src/components/posthog-provider.tsx", + "src/lib/analytics.ts", + "src/lib/vitals.ts", + "src/pages/HomeClient.tsx", + "tests/open-app.spec.ts" + ], + "recall_at_5": 0.125, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.2, + "recall_at_20": 0.25, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.125, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 4, + "tokens_delivered": 4759, + "latency_ms": 1530.652, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-3c474e85a025", + "path_leak": true, + "required_files": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "found": [], + "missed": [ + "landing-astro/src/pages/faq.astro", + "landing-astro/src/pages/index.astro", + "src/components/SignInScreen.tsx", + "src/pages/PrivacyPage.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2419, + "latency_ms": 1941.85, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-3cbd63102c96", + "path_leak": true, + "required_files": [ + ".husky/pre-push" + ], + "found": [ + ".husky/pre-push" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2212, + "latency_ms": 748.259, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-45ab8efac77e", + "path_leak": false, + "required_files": [ + "src/components/EmailDetail.tsx" + ], + "found": [ + "src/components/EmailDetail.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 5842, + "latency_ms": 1355.368, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-4f284d35f52f", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 600, + "latency_ms": 751.272, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-50270cf8bf36", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [ + "src/components/EmailList.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1679, + "latency_ms": 108.3, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-5dd4caedcd8d", + "path_leak": true, + "required_files": [ + "src/lib/auth.ts" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4690, + "latency_ms": 1000.04, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-5e56f1efdc60", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 598, + "latency_ms": 851.84, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-61a651298526", + "path_leak": true, + "required_files": [ + "src/components/EmailList.tsx" + ], + "found": [ + "src/components/EmailList.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1679, + "latency_ms": 119.109, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-63eecd0ec045", + "path_leak": false, + "required_files": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts", + "worker.mjs" + ], + "found": [ + "src/app/app/HomeClient.tsx", + "src/lib/get-access-token.ts" + ], + "missed": [ + "worker.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 6, + "tokens_delivered": 1677, + "latency_ms": 1196.506, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-64def25acbb3", + "path_leak": true, + "required_files": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "src/lib/auth.ts", + "wrangler.toml" + ], + "found": [ + "src/lib/auth.ts" + ], + "missed": [ + "migrations/0001_better_auth.sql", + "src/db/schema.ts", + "wrangler.toml" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1667, + "recall_at_20": 0.25, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 1388, + "latency_ms": 795.49, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-6acde8cf1b54", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1036, + "latency_ms": 927.975, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-6cf000313340", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1033, + "latency_ms": 716.803, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-7796afb6ea63", + "path_leak": false, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6957, + "latency_ms": 636.129, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-7b0fe5379d1a", + "path_leak": false, + "required_files": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/api/emails/route.ts", + "src/app/page.tsx", + "src/lib/auth.ts" + ], + "found": [ + "src/app/api/emails/route.ts", + "src/lib/auth.ts" + ], + "missed": [ + ".gitignore", + "src/app/api/emails/[id]/route.ts", + "src/app/page.tsx" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.4, + "recall_at_20": 0.4, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 3, + "tokens_delivered": 1418, + "latency_ms": 603.998, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-88e6e8039b06", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 895, + "latency_ms": 2218.041, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-8a3b6110595c", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 1687, + "latency_ms": 902.375, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-92f5176080c2", + "path_leak": false, + "required_files": [ + "src/app/layout.tsx" + ], + "found": [ + "src/app/layout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 665, + "latency_ms": 791.064, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-95a35f10d8b7", + "path_leak": true, + "required_files": [ + "src/worker.ts", + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "found": [ + "src/worker.ts" + ], + "missed": [ + "tests/open-app.spec.ts", + "wrangler.toml" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 1690, + "latency_ms": 1729, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-ade59255e434", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2022, + "latency_ms": 3657.537, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-af92947d28e7", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 96, + "latency_ms": 906.928, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-b3822a197c25", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1684, + "latency_ms": 1041.492, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-b5e31e012879", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1288, + "latency_ms": 815.782, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-ba9e6497f9eb", + "path_leak": false, + "required_files": [ + "next.config.ts" + ], + "found": [], + "missed": [ + "next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7, + "latency_ms": 673.104, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-c64f6290bdd8", + "path_leak": true, + "required_files": [ + "wrangler.toml" + ], + "found": [], + "missed": [ + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 314, + "latency_ms": 915.738, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-c7d7703e1f35", + "path_leak": false, + "required_files": [ + "src/app/page.tsx" + ], + "found": [ + "src/app/page.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 14, + "tokens_delivered": 1685, + "latency_ms": 914.728, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-cd131b24cb29", + "path_leak": true, + "required_files": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "found": [], + "missed": [ + "index.html", + "landing-astro/src/layouts/Layout.astro", + "scripts/verify-public-indexing.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2633, + "latency_ms": 1868.778, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-d1746f53d55e", + "path_leak": false, + "required_files": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "found": [], + "missed": [ + "public/_headers", + "src/lib/security-headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2533, + "latency_ms": 1609.239, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-e1bbe3e69163", + "path_leak": false, + "required_files": [ + "src/lib/security-headers.ts", + "src/worker.ts" + ], + "found": [ + "src/worker.ts" + ], + "missed": [ + "src/lib/security-headers.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1679, + "latency_ms": 1286.149, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-e52fcd58cd2d", + "path_leak": true, + "required_files": [ + "src/lib/gmail.ts" + ], + "found": [ + "src/lib/gmail.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 152, + "latency_ms": 593.171, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-edc4fd5d594b", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1275, + "latency_ms": 622.852, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-f1c2f027bafe", + "path_leak": false, + "required_files": [ + "tests/example.spec.ts" + ], + "found": [ + "tests/example.spec.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 2826, + "latency_ms": 1176.089, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "email-manager-f24464fd6750", + "path_leak": true, + "required_files": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css", + "src/worker.ts" + ], + "found": [ + "src/components/EmailDetail.tsx", + "src/components/TriageQueues.tsx", + "src/components/TriageSession.tsx", + "src/worker.ts" + ], + "missed": [ + "src/components/ui/badge.tsx", + "src/lib/email-html.ts", + "src/lib/format-date.ts", + "src/styles/globals.css" + ], + "recall_at_5": 0.125, + "precision_at_5": 0.2, + "recall_at_10": 0.125, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1662, + "latency_ms": 1439.87, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/V-protein-index.json b/benchmarks/context-retrieval/results/V-protein-index.json new file mode 100644 index 00000000..5ae1ce13 --- /dev/null +++ b/benchmarks/context-retrieval/results/V-protein-index.json @@ -0,0 +1,6509 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "protein-index", + "head": "e96ee7fd3164bee1493925dcba8a160c5046e1d8" + }, + "tier": "small", + "tier_code_files": 104, + "corpus_counts": { + "cases": 52, + "path_leak": 11, + "baseline_blind": 41, + "multi_file": 35 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.6228, + "control": 0.0144, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "reject volume-basis label candidates", + "revision": "c07e964dd0d26f06c01dbaf0b5f47c94aa87ff05", + "returned": [ + "shared/classification.ts", + "test/api.worker.test.ts", + "migrations/0005_evidence_decisions.sql", + "openspec/changes/build-catalog-core/.openspec.yaml", + "review-decisions/review-615f9e122d922268afd3/manifest.json" + ] + }, + { + "provider_id": "random-files", + "query": "preserve label sugar and calorie semantics", + "revision": "e79743cf9d782a065e0e4a54720d8c5c14982026", + "returned": [ + ".github/workflows/extract-robotoff-ingredients.yml", + "scripts/adapters/robotoff.ts", + "review-decisions/review-4c07ab1d3adc20a99ccb/manifest.json", + "scripts/adapters/robotoff-ingredients.ts", + "openspec/config.yaml" + ] + }, + { + "provider_id": "random-files", + "query": "publish source catalog to D1", + "revision": "c339b464b206cb32f5966ec8240c0e79375f9674", + "returned": [ + "scripts/fixtures.ts", + "test/domain.test.ts", + "vite.config.ts", + "package.json", + "worker/catalog.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "reject volume-basis label candidates", + "revision": "c07e964dd0d26f06c01dbaf0b5f47c94aa87ff05", + "returned": [ + "test/ingestion.test.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "preserve unknown nutrition basis", + "revision": "392d519207f9163a413d8e09d37305866f595067", + "returned": [ + "test/ingestion.test.ts", + "scripts/reconcile.ts", + "src/App.tsx", + ".github/workflows/source-sync.yml", + "scripts/adapters/open-food-facts-api.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "select current review evidence", + "revision": "cadcf1bd67cfddbe126fb06f5204e180e5f25be4", + "returned": [ + "test/ingestion.test.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "scripts/reconcile.ts", + "src/App.tsx" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "reject volume-basis label candidates", + "revision": "c07e964dd0d26f06c01dbaf0b5f47c94aa87ff05", + "returned": [ + "test/ingestion.test.ts", + "test/domain.test.ts", + "openspec/changes/add-label-evidence-extraction/specs/evidence-aware-protein-ranking/spec.md", + "scripts/sync.ts", + "scripts/adapters/robotoff.ts" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "preserve label sugar and calorie semantics", + "revision": "e79743cf9d782a065e0e4a54720d8c5c14982026", + "returned": [ + "test/domain.test.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts", + "shared/api.ts" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 52, + "mean_recall_at_5": 0.0503, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1006, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1638, + "full_recall_at_20": 0.0385, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0096, + "mean_recall_at_16000_tokens": 0.0407, + "mean_precision_at_10": 0.0288, + "zero_hit_rate": 0.6538, + "median_tokens_delivered": 40103.5, + "median_latency_ms": 12.2495, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 52, + "mean_recall_at_5": 0.0503, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1006, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1638, + "full_recall_at_20": 0.0385, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0096, + "mean_recall_at_16000_tokens": 0.0407, + "mean_precision_at_10": 0.0288, + "zero_hit_rate": 0.6538, + "median_tokens_delivered": 40103.5, + "median_latency_ms": 12.2495, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 11, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0152, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0606, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0152, + "mean_precision_at_10": 0.0091, + "zero_hit_rate": 0.7273, + "median_tokens_delivered": 43785, + "median_latency_ms": 12.468, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 41, + "mean_recall_at_5": 0.0638, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1236, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1915, + "full_recall_at_20": 0.0488, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0122, + "mean_recall_at_16000_tokens": 0.0476, + "mean_precision_at_10": 0.0341, + "zero_hit_rate": 0.6341, + "median_tokens_delivered": 40044, + "median_latency_ms": 12.086, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 52 + }, + "resources": { + "baseline_rss_mb": 27072.2, + "peak_rss_mb": 27177.8, + "delta_rss_mb": 105.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 52, + "mean_recall_at_5": 0.3603, + "full_recall_at_5": 0.1538, + "mean_recall_at_10": 0.4788, + "full_recall_at_10": 0.2308, + "mean_recall_at_20": 0.5651, + "full_recall_at_20": 0.3462, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0144, + "mean_recall_at_16000_tokens": 0.2883, + "mean_precision_at_10": 0.1346, + "zero_hit_rate": 0.2885, + "median_tokens_delivered": 145928, + "median_latency_ms": 21.4675, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 52, + "mean_recall_at_5": 0.3603, + "full_recall_at_5": 0.1538, + "mean_recall_at_10": 0.4788, + "full_recall_at_10": 0.2308, + "mean_recall_at_20": 0.5651, + "full_recall_at_20": 0.3462, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0144, + "mean_recall_at_16000_tokens": 0.2883, + "mean_precision_at_10": 0.1346, + "zero_hit_rate": 0.2885, + "median_tokens_delivered": 145928, + "median_latency_ms": 21.4675, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 11, + "mean_recall_at_5": 0.2682, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.3515, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.4727, + "full_recall_at_20": 0.1818, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.2076, + "mean_precision_at_10": 0.1364, + "zero_hit_rate": 0.2727, + "median_tokens_delivered": 189454, + "median_latency_ms": 22.009, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 41, + "mean_recall_at_5": 0.385, + "full_recall_at_5": 0.1951, + "mean_recall_at_10": 0.513, + "full_recall_at_10": 0.2927, + "mean_recall_at_20": 0.5898, + "full_recall_at_20": 0.3902, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0183, + "mean_recall_at_16000_tokens": 0.31, + "mean_precision_at_10": 0.1341, + "zero_hit_rate": 0.2927, + "median_tokens_delivered": 140313, + "median_latency_ms": 21.298, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 52 + }, + "resources": { + "baseline_rss_mb": 27181.6, + "peak_rss_mb": 27355.9, + "delta_rss_mb": 174.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 52, + "mean_recall_at_5": 0.2838, + "full_recall_at_5": 0.1154, + "mean_recall_at_10": 0.4556, + "full_recall_at_10": 0.2692, + "mean_recall_at_20": 0.5902, + "full_recall_at_20": 0.3654, + "mean_recall_at_1000_tokens": 0.5575, + "mean_recall_at_4000_tokens": 0.5902, + "mean_recall_at_16000_tokens": 0.5902, + "mean_precision_at_10": 0.1196, + "zero_hit_rate": 0.2308, + "median_tokens_delivered": 904, + "median_latency_ms": 648.196, + "payload_kind": [ + "excerpts" + ], + "unavailable": 3 + }, + "baseline_missed": { + "cases": 52, + "mean_recall_at_5": 0.2838, + "full_recall_at_5": 0.1154, + "mean_recall_at_10": 0.4556, + "full_recall_at_10": 0.2692, + "mean_recall_at_20": 0.5902, + "full_recall_at_20": 0.3654, + "mean_recall_at_1000_tokens": 0.5575, + "mean_recall_at_4000_tokens": 0.5902, + "mean_recall_at_16000_tokens": 0.5902, + "mean_precision_at_10": 0.1196, + "zero_hit_rate": 0.2308, + "median_tokens_delivered": 904, + "median_latency_ms": 648.196, + "payload_kind": [ + "excerpts" + ], + "unavailable": 3 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 11, + "mean_recall_at_5": 0.3364, + "full_recall_at_5": 0.1818, + "mean_recall_at_10": 0.5182, + "full_recall_at_10": 0.3636, + "mean_recall_at_20": 0.6545, + "full_recall_at_20": 0.4545, + "mean_recall_at_1000_tokens": 0.5636, + "mean_recall_at_4000_tokens": 0.6545, + "mean_recall_at_16000_tokens": 0.6545, + "mean_precision_at_10": 0.1273, + "zero_hit_rate": 0.0909, + "median_tokens_delivered": 1177, + "median_latency_ms": 738.742, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 41, + "mean_recall_at_5": 0.2697, + "full_recall_at_5": 0.0976, + "mean_recall_at_10": 0.4388, + "full_recall_at_10": 0.2439, + "mean_recall_at_20": 0.573, + "full_recall_at_20": 0.3415, + "mean_recall_at_1000_tokens": 0.5559, + "mean_recall_at_4000_tokens": 0.573, + "mean_recall_at_16000_tokens": 0.573, + "mean_precision_at_10": 0.1175, + "zero_hit_rate": 0.2683, + "median_tokens_delivered": 855, + "median_latency_ms": 636.205, + "payload_kind": [ + "excerpts" + ], + "unavailable": 3 + } + }, + "outcomes": { + "answered": 49, + "refused-own-capacity-limit": 3 + }, + "resources": { + "baseline_rss_mb": 27350.6, + "peak_rss_mb": 28818.5, + "delta_rss_mb": 1467.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 1164185600 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 52, + "mean_recall_at_5": 0.2995, + "full_recall_at_5": 0.1346, + "mean_recall_at_10": 0.4684, + "full_recall_at_10": 0.25, + "mean_recall_at_20": 0.6468, + "full_recall_at_20": 0.4615, + "mean_recall_at_1000_tokens": 0.4159, + "mean_recall_at_4000_tokens": 0.6228, + "mean_recall_at_16000_tokens": 0.6468, + "mean_precision_at_10": 0.1264, + "zero_hit_rate": 0.2115, + "median_tokens_delivered": 1694, + "median_latency_ms": 2213.5625, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 52, + "mean_recall_at_5": 0.2995, + "full_recall_at_5": 0.1346, + "mean_recall_at_10": 0.4684, + "full_recall_at_10": 0.25, + "mean_recall_at_20": 0.6468, + "full_recall_at_20": 0.4615, + "mean_recall_at_1000_tokens": 0.4159, + "mean_recall_at_4000_tokens": 0.6228, + "mean_recall_at_16000_tokens": 0.6468, + "mean_precision_at_10": 0.1264, + "zero_hit_rate": 0.2115, + "median_tokens_delivered": 1694, + "median_latency_ms": 2213.5625, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 11, + "mean_recall_at_5": 0.397, + "full_recall_at_5": 0.1818, + "mean_recall_at_10": 0.5742, + "full_recall_at_10": 0.2727, + "mean_recall_at_20": 0.6727, + "full_recall_at_20": 0.4545, + "mean_recall_at_1000_tokens": 0.5894, + "mean_recall_at_4000_tokens": 0.6727, + "mean_recall_at_16000_tokens": 0.6727, + "mean_precision_at_10": 0.1818, + "zero_hit_rate": 0.1818, + "median_tokens_delivered": 1690, + "median_latency_ms": 2553.103, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 41, + "mean_recall_at_5": 0.2734, + "full_recall_at_5": 0.122, + "mean_recall_at_10": 0.44, + "full_recall_at_10": 0.2439, + "mean_recall_at_20": 0.6398, + "full_recall_at_20": 0.4634, + "mean_recall_at_1000_tokens": 0.3693, + "mean_recall_at_4000_tokens": 0.6093, + "mean_recall_at_16000_tokens": 0.6398, + "mean_precision_at_10": 0.1116, + "zero_hit_rate": 0.2195, + "median_tokens_delivered": 1695, + "median_latency_ms": 2148.848, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 52 + }, + "resources": { + "baseline_rss_mb": 26704.4, + "peak_rss_mb": 28743.2, + "delta_rss_mb": 2038.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 152801280 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "protein-index-02bae53a54b2", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "vitest.worker.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 209011, + "latency_ms": 26.523, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-12524e50831f", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28922, + "latency_ms": 11.533, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-14a6c420d60c", + "path_leak": true, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13328, + "latency_ms": 14.675, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-183e0a12fb52", + "path_leak": true, + "required_files": [ + "scripts/machine-label.ts" + ], + "found": [], + "missed": [ + "scripts/machine-label.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 192434, + "latency_ms": 15.57, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-196a41597b98", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/domain.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 163973, + "latency_ms": 13.114, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-1a649b8e8091", + "path_leak": false, + "required_files": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40044, + "latency_ms": 13.009, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-1af57b34dd19", + "path_leak": true, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63853, + "latency_ms": 12.491, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-1e4d6787f0ec", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 167953, + "latency_ms": 12.046, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-22fa292f8766", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21367, + "latency_ms": 11.349, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-23339dc6c7ac", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36271, + "latency_ms": 12.389, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-254d8a38d40a", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 26995, + "latency_ms": 11.593, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-2bdf658143b0", + "path_leak": true, + "required_files": [ + "scripts/official-brand-publication.ts" + ], + "found": [], + "missed": [ + "scripts/official-brand-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26217, + "latency_ms": 12.468, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-3b759f36734b", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "found": [ + "scripts/reconcile.ts" + ], + "missed": [ + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 49463, + "latency_ms": 12.37, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-3c87dcc8228f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 66382, + "latency_ms": 11.039, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-3f67cd6936fb", + "path_leak": false, + "required_files": [ + "test/official-brand-sitemap.test.ts" + ], + "found": [], + "missed": [ + "test/official-brand-sitemap.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19886, + "latency_ms": 11.474, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-3fa2c53fa804", + "path_leak": false, + "required_files": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 52206, + "latency_ms": 11.692, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-40d2e4ea219e", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20279, + "latency_ms": 11.077, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-42eba2b71c9f", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40163, + "latency_ms": 11.899, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-52a00772a842", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26209, + "latency_ms": 12.207, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-54927ef9d7e2", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 11377, + "latency_ms": 12.086, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-5c62a39276da", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 36849, + "latency_ms": 11.105, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-6c2b66ff7837", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39039, + "latency_ms": 12.981, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-71d860be5034", + "path_leak": false, + "required_files": [ + "test/manual-publication-gate.test.ts" + ], + "found": [], + "missed": [ + "test/manual-publication-gate.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13728, + "latency_ms": 12.988, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-755dcb7a8906", + "path_leak": true, + "required_files": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 61473, + "latency_ms": 12.989, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-796f6b736144", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "found": [ + "scripts/review-bundles.ts" + ], + "missed": [ + "scripts/reconcile.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 46940, + "latency_ms": 13.977, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-880bc84a1840", + "path_leak": false, + "required_files": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "found": [], + "missed": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 176705, + "latency_ms": 12.423, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-88583294807f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51254, + "latency_ms": 13.077, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-8d4a8954b40e", + "path_leak": true, + "required_files": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "test/api.worker.test.ts" + ], + "missed": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 57760, + "latency_ms": 10.945, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-970e6366ccb9", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 17363, + "latency_ms": 11.294, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-985886cf2935", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 191083, + "latency_ms": 11.448, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-9b8659e2671b", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts" + ], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 51662, + "latency_ms": 12.314, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-a580a089b8cc", + "path_leak": true, + "required_files": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts" + ], + "missed": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 10, + "tokens_delivered": 27773, + "latency_ms": 12.456, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-b603f405353b", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26359, + "latency_ms": 11.42, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-b7e0001c5df7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27265, + "latency_ms": 11.736, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-be2529dd2357", + "path_leak": true, + "required_files": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "found": [ + "scripts/review-bundles.ts" + ], + "missed": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 43785, + "latency_ms": 10.921, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-c07e964dd0d2", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 201348, + "latency_ms": 12.715, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-c1ab8fe455d4", + "path_leak": true, + "required_files": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25632, + "latency_ms": 11.43, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-c634f70476c4", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32441, + "latency_ms": 11.905, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-c725a3c50f27", + "path_leak": false, + "required_files": [ + "src/App.tsx" + ], + "found": [ + "src/App.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 53078, + "latency_ms": 11.118, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-cadcf1bd67cf", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53150, + "latency_ms": 11.842, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-cb4efe626dc5", + "path_leak": false, + "required_files": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 180990, + "latency_ms": 12.501, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-d131feb0b34a", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33820, + "latency_ms": 11.816, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-d2003fb0158f", + "path_leak": true, + "required_files": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "found": [], + "missed": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49602, + "latency_ms": 12.375, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-d3b0639e3065", + "path_leak": false, + "required_files": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37735, + "latency_ms": 11.561, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-d3e9f665abd0", + "path_leak": false, + "required_files": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [], + "missed": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37975, + "latency_ms": 11.977, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-df42c030ac4d", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 35776, + "latency_ms": 13.202, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-e9b06828e033", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 46805, + "latency_ms": 12.292, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-e9dff924a117", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "found": [ + "test/domain.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 180693, + "latency_ms": 13.924, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-ee7d155b7b9e", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17672, + "latency_ms": 12.132, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-f4da150e6ba9", + "path_leak": true, + "required_files": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17339, + "latency_ms": 13.481, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-f9a264cd2de7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48848, + "latency_ms": 13.796, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "protein-index-fda927717fdd", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 48561, + "latency_ms": 13.489, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-02bae53a54b2", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 109279, + "latency_ms": 20.003, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-12524e50831f", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 113847, + "latency_ms": 22.547, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-14a6c420d60c", + "path_leak": true, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 146146, + "latency_ms": 22.297, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-183e0a12fb52", + "path_leak": true, + "required_files": [ + "scripts/machine-label.ts" + ], + "found": [], + "missed": [ + "scripts/machine-label.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 243078, + "latency_ms": 26.302, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-196a41597b98", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "shared/nutrition.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.8, + "precision_at_10": 0.4, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 61735, + "latency_ms": 17.132, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-1a649b8e8091", + "path_leak": false, + "required_files": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.4, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.125, + "first_hit_rank": 1, + "tokens_delivered": 108324, + "latency_ms": 16.71, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-1af57b34dd19", + "path_leak": true, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 242323, + "latency_ms": 24.221, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-1e4d6787f0ec", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 188187, + "latency_ms": 23.467, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-22fa292f8766", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 111543, + "latency_ms": 20.145, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-23339dc6c7ac", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.8, + "precision_at_10": 0.4, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 1, + "tokens_delivered": 125200, + "latency_ms": 23.315, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-254d8a38d40a", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 145873, + "latency_ms": 24.308, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-2bdf658143b0", + "path_leak": true, + "required_files": [ + "scripts/official-brand-publication.ts" + ], + "found": [], + "missed": [ + "scripts/official-brand-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 242323, + "latency_ms": 22.009, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-3b759f36734b", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "found": [ + "scripts/reconcile.ts" + ], + "missed": [ + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 240039, + "latency_ms": 23.077, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-3c87dcc8228f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 242323, + "latency_ms": 23.514, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-3f67cd6936fb", + "path_leak": false, + "required_files": [ + "test/official-brand-sitemap.test.ts" + ], + "found": [], + "missed": [ + "test/official-brand-sitemap.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 242323, + "latency_ms": 21.637, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-3fa2c53fa804", + "path_leak": false, + "required_files": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 243078, + "latency_ms": 22.016, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-40d2e4ea219e", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 112625, + "latency_ms": 19.114, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-42eba2b71c9f", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 232569, + "latency_ms": 22.682, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-52a00772a842", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 59582, + "latency_ms": 18.112, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-54927ef9d7e2", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 255550, + "latency_ms": 23.675, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-5c62a39276da", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 61748, + "latency_ms": 15.917, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-6c2b66ff7837", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 241591, + "latency_ms": 25.191, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-71d860be5034", + "path_leak": false, + "required_files": [ + "test/manual-publication-gate.test.ts" + ], + "found": [], + "missed": [ + "test/manual-publication-gate.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 240039, + "latency_ms": 21.659, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-755dcb7a8906", + "path_leak": true, + "required_files": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/publication.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 104763, + "latency_ms": 18.935, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-796f6b736144", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "found": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "missed": [], + "recall_at_5": 0.6, + "precision_at_5": 0.6, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 1, + "tokens_delivered": 139291, + "latency_ms": 19.072, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-880bc84a1840", + "path_leak": false, + "required_files": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "found": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 105620, + "latency_ms": 16.08, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-88583294807f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 241591, + "latency_ms": 23.15, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-8d4a8954b40e", + "path_leak": true, + "required_files": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "shared/classification.ts", + "shared/metrics.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 105144, + "latency_ms": 17.11, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-970e6366ccb9", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 124661, + "latency_ms": 19.624, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-985886cf2935", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "shared/types.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 61708, + "latency_ms": 15.708, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-9b8659e2671b", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/robotoff.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 105882, + "latency_ms": 16.253, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-a580a089b8cc", + "path_leak": true, + "required_files": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "migrations/0007_review_queue_indexes.sql", + "shared/gtin.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 1, + "tokens_delivered": 109431, + "latency_ms": 18.253, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-b603f405353b", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/robotoff-ingredients-api.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 145983, + "latency_ms": 21.939, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-b7e0001c5df7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 240039, + "latency_ms": 20.081, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-be2529dd2357", + "path_leak": true, + "required_files": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "found": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 240119, + "latency_ms": 20.245, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-c07e964dd0d2", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 109089, + "latency_ms": 19.539, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-c1ab8fe455d4", + "path_leak": true, + "required_files": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-ingredients-api.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 189454, + "latency_ms": 24.403, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-c634f70476c4", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 232984, + "latency_ms": 22.964, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-c725a3c50f27", + "path_leak": false, + "required_files": [ + "src/App.tsx" + ], + "found": [ + "src/App.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 240159, + "latency_ms": 21.298, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-cadcf1bd67cf", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 233369, + "latency_ms": 22.853, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-cb4efe626dc5", + "path_leak": false, + "required_files": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/review-bundles.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 130024, + "latency_ms": 20.676, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-d131feb0b34a", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 240039, + "latency_ms": 21.154, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-d2003fb0158f", + "path_leak": true, + "required_files": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "found": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 104923, + "latency_ms": 15.926, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-d3b0639e3065", + "path_leak": false, + "required_files": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 140225, + "latency_ms": 18.708, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-d3e9f665abd0", + "path_leak": false, + "required_files": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "shared/nutrition.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 62240, + "latency_ms": 16.702, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-df42c030ac4d", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 146974, + "latency_ms": 22.472, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-e9b06828e033", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 55044, + "latency_ms": 15.799, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-e9dff924a117", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "test/domain.test.ts" + ], + "missed": [ + "shared/ingredients.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 50889, + "latency_ms": 16.426, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-ee7d155b7b9e", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 242323, + "latency_ms": 46.884, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-f4da150e6ba9", + "path_leak": true, + "required_files": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 233497, + "latency_ms": 26.965, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-f9a264cd2de7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 240039, + "latency_ms": 26.349, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "protein-index-fda927717fdd", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 140313, + "latency_ms": 21.646, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-02bae53a54b2", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "vitest.worker.config.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 906, + "latency_ms": 435.783, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-12524e50831f", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1863, + "latency_ms": 487.981, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-14a6c420d60c", + "path_leak": true, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1579, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1624, + "latency_ms": 548.801, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-183e0a12fb52", + "path_leak": true, + "required_files": [ + "scripts/machine-label.ts" + ], + "found": [ + "scripts/machine-label.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 743, + "latency_ms": 827.549, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-196a41597b98", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "shared/nutrition.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.8, + "precision_at_20": 0.2105, + "recall_at_1000_tokens": 0.8, + "recall_at_4000_tokens": 0.8, + "recall_at_16000_tokens": 0.8, + "first_hit_rank": 2, + "tokens_delivered": 689, + "latency_ms": 368.298, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-1a649b8e8091", + "path_leak": false, + "required_files": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/robotoff-api.ts" + ], + "recall_at_5": 0.125, + "precision_at_5": 0.2, + "recall_at_10": 0.375, + "precision_at_10": 0.3, + "recall_at_20": 0.625, + "precision_at_20": 0.2632, + "recall_at_1000_tokens": 0.625, + "recall_at_4000_tokens": 0.625, + "recall_at_16000_tokens": 0.625, + "first_hit_rank": 1, + "tokens_delivered": 658, + "latency_ms": 438.433, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-1af57b34dd19", + "path_leak": true, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "found": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "missed": [ + "config/official-brand-sources.json", + "scripts/official-brand-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 19, + "tokens_delivered": 1585, + "latency_ms": 783.395, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-1e4d6787f0ec", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 464, + "latency_ms": 606.279, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-22fa292f8766", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/sync.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 748, + "latency_ms": 445.779, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-23339dc6c7ac", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "found": [ + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 17, + "tokens_delivered": 819, + "latency_ms": 656.093, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-254d8a38d40a", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 1037, + "latency_ms": 744.322, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-2bdf658143b0", + "path_leak": true, + "required_files": [ + "scripts/official-brand-publication.ts" + ], + "found": [ + "scripts/official-brand-publication.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1134, + "latency_ms": 1115.434, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-3b759f36734b", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "found": [ + "scripts/reconcile.ts" + ], + "missed": [ + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 946, + "latency_ms": 902.235, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-3c87dcc8228f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 926, + "latency_ms": 1182.299, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-3f67cd6936fb", + "path_leak": false, + "required_files": [ + "test/official-brand-sitemap.test.ts" + ], + "found": [ + "test/official-brand-sitemap.test.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 855, + "latency_ms": 861.373, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-3fa2c53fa804", + "path_leak": false, + "required_files": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1780, + "latency_ms": 1068.809, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-40d2e4ea219e", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 623, + "latency_ms": 481.991, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-42eba2b71c9f", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1415, + "latency_ms": 938.8, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-52a00772a842", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 487, + "latency_ms": 488.122, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-54927ef9d7e2", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 636.205, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-5c62a39276da", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 352, + "latency_ms": 608.814, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-6c2b66ff7837", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 663.843, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-71d860be5034", + "path_leak": false, + "required_files": [ + "test/manual-publication-gate.test.ts" + ], + "found": [], + "missed": [ + "test/manual-publication-gate.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1063, + "latency_ms": 950.45, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-755dcb7a8906", + "path_leak": true, + "required_files": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1177, + "latency_ms": 583.02, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-796f6b736144", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "found": [ + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "missed": [ + "scripts/reconcile.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.6, + "recall_at_4000_tokens": 0.8, + "recall_at_16000_tokens": 0.8, + "first_hit_rank": 1, + "tokens_delivered": 1407, + "latency_ms": 538.329, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-880bc84a1840", + "path_leak": false, + "required_files": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "found": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 831, + "latency_ms": 451.097, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-88583294807f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 804.452, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-8d4a8954b40e", + "path_leak": true, + "required_files": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "shared/metrics.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "shared/classification.ts" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.8333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.8333, + "recall_at_4000_tokens": 0.8333, + "recall_at_16000_tokens": 0.8333, + "first_hit_rank": 5, + "tokens_delivered": 1050, + "latency_ms": 506.943, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-970e6366ccb9", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/robotoff.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1222, + "latency_ms": 528.573, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-985886cf2935", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 892, + "latency_ms": 547.446, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-9b8659e2671b", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/robotoff.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 787, + "latency_ms": 1039.428, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-a580a089b8cc", + "path_leak": true, + "required_files": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/reconcile.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "shared/gtin.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3481, + "latency_ms": 482.273, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-b603f405353b", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1782, + "latency_ms": 502.915, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-b7e0001c5df7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 516, + "latency_ms": 924.674, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-be2529dd2357", + "path_leak": true, + "required_files": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "found": [ + "scripts/review-bundles.ts" + ], + "missed": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 17, + "tokens_delivered": 2007, + "latency_ms": 1057.8, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-c07e964dd0d2", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 816, + "latency_ms": 416.701, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-c1ab8fe455d4", + "path_leak": true, + "required_files": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 863, + "latency_ms": 738.742, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-c634f70476c4", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1215, + "latency_ms": 852.49, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-c725a3c50f27", + "path_leak": false, + "required_files": [ + "src/App.tsx" + ], + "found": [], + "missed": [ + "src/App.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1279, + "latency_ms": 687.013, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-cadcf1bd67cf", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 359, + "latency_ms": 884.15, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-cb4efe626dc5", + "path_leak": false, + "required_files": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/review-bundles.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 1009, + "latency_ms": 628.702, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-d131feb0b34a", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [], + "missed": [ + "scripts/guarded-publication.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 601, + "latency_ms": 886.933, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-d2003fb0158f", + "path_leak": true, + "required_files": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "found": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 589, + "latency_ms": 658.386, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-d3b0639e3065", + "path_leak": false, + "required_files": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 942, + "latency_ms": 632.627, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-d3e9f665abd0", + "path_leak": false, + "required_files": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "shared/nutrition.ts", + "test/domain.test.ts" + ], + "missed": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2857, + "recall_at_20": 0.5, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 226, + "latency_ms": 640.299, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-df42c030ac4d", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 1100, + "latency_ms": 525.758, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-e9b06828e033", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 660, + "latency_ms": 482.401, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-e9dff924a117", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "test/domain.test.ts" + ], + "missed": [ + "shared/ingredients.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 10, + "tokens_delivered": 503, + "latency_ms": 556.457, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-ee7d155b7b9e", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1668, + "latency_ms": 795.007, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-f4da150e6ba9", + "path_leak": true, + "required_files": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 4, + "tokens_delivered": 1992, + "latency_ms": 987.597, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-f9a264cd2de7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [ + "scripts/guarded-publication.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 1144, + "latency_ms": 784.822, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "protein-index-fda927717fdd", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 902, + "latency_ms": 743.184, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-02bae53a54b2", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts", + "vitest.worker.config.ts" + ], + "missed": [ + "scripts/adapters/robotoff.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 12, + "tokens_delivered": 1695, + "latency_ms": 2501.302, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-12524e50831f", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 1683, + "latency_ms": 2662.158, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-14a6c420d60c", + "path_leak": true, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1691, + "latency_ms": 4521.633, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-183e0a12fb52", + "path_leak": true, + "required_files": [ + "scripts/machine-label.ts" + ], + "found": [], + "missed": [ + "scripts/machine-label.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1690, + "latency_ms": 5348.192, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-196a41597b98", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "shared/nutrition.ts", + "test/api.worker.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 6, + "tokens_delivered": 1691, + "latency_ms": 1959.131, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-1a649b8e8091", + "path_leak": false, + "required_files": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json", + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "scripts/adapters/robotoff-api.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/review-66191036dc5b4534f422/checksums.sha256", + "review-decisions/review-66191036dc5b4534f422/manifest.json" + ], + "recall_at_5": 0.375, + "precision_at_5": 0.6, + "recall_at_10": 0.625, + "precision_at_10": 0.5, + "recall_at_20": 0.75, + "precision_at_20": 0.3, + "recall_at_1000_tokens": 0.625, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 1681, + "latency_ms": 2148.848, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-1af57b34dd19", + "path_leak": true, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "found": [ + "scripts/adapters/official-brand-sitemap.ts", + "scripts/official-brand-publication.ts" + ], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 1661, + "latency_ms": 4511.601, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-1e4d6787f0ec", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1204, + "latency_ms": 2955.359, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-22fa292f8766", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/sync.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1683, + "latency_ms": 1790.45, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-23339dc6c7ac", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts" + ], + "missed": [ + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/catalog.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 8, + "tokens_delivered": 4817, + "latency_ms": 2298.65, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-254d8a38d40a", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1708, + "latency_ms": 1931.877, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-2bdf658143b0", + "path_leak": true, + "required_files": [ + "scripts/official-brand-publication.ts" + ], + "found": [ + "scripts/official-brand-publication.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1703, + "latency_ms": 4273.926, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-3b759f36734b", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "found": [ + "scripts/reconcile.ts" + ], + "missed": [ + "test/reconcile-terminal-evidence-replay.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 1699, + "latency_ms": 3248.433, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-3c87dcc8228f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1680, + "latency_ms": 3592.542, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-3f67cd6936fb", + "path_leak": false, + "required_files": [ + "test/official-brand-sitemap.test.ts" + ], + "found": [], + "missed": [ + "test/official-brand-sitemap.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1673, + "latency_ms": 2434.801, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-3fa2c53fa804", + "path_leak": false, + "required_files": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 1681, + "latency_ms": 2393.815, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-40d2e4ea219e", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1668, + "latency_ms": 2971.481, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-42eba2b71c9f", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 8016, + "latency_ms": 2043.991, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-52a00772a842", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5792, + "latency_ms": 1211.141, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-54927ef9d7e2", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1687, + "latency_ms": 3523.1, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-5c62a39276da", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 6768, + "latency_ms": 1168.897, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-6c2b66ff7837", + "path_leak": false, + "required_files": [ + "public/_headers" + ], + "found": [], + "missed": [ + "public/_headers" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2879, + "latency_ms": 2733.763, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-71d860be5034", + "path_leak": false, + "required_files": [ + "test/manual-publication-gate.test.ts" + ], + "found": [], + "missed": [ + "test/manual-publication-gate.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 9232, + "latency_ms": 2130.511, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-755dcb7a8906", + "path_leak": true, + "required_files": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/publication.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1690, + "latency_ms": 1179.061, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-796f6b736144", + "path_leak": false, + "required_files": [ + "scripts/reconcile.ts", + "scripts/review-bundles.ts", + "test/api.worker.test.ts", + "test/ingestion.test.ts", + "worker/reviews.ts" + ], + "found": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/reconcile.ts", + "test/api.worker.test.ts", + "worker/reviews.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 16, + "tokens_delivered": 1697, + "latency_ms": 3029.035, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-880bc84a1840", + "path_leak": false, + "required_files": [ + "test/api.worker.test.ts", + "worker/catalog.ts" + ], + "found": [ + "worker/catalog.ts" + ], + "missed": [ + "test/api.worker.test.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 581, + "latency_ms": 1293.625, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-88583294807f", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json", + "scripts/adapters/official-brand-sitemap.ts" + ], + "found": [ + "scripts/adapters/official-brand-sitemap.ts" + ], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1691, + "latency_ms": 2951.522, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-8d4a8954b40e", + "path_leak": true, + "required_files": [ + "shared/classification.ts", + "shared/metrics.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "shared/metrics.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "shared/classification.ts", + "shared/nutrition.ts", + "test/api.worker.test.ts" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1697, + "latency_ms": 1399.718, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-970e6366ccb9", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1723, + "latency_ms": 1576.196, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-985886cf2935", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "shared/types.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1695, + "latency_ms": 1038.516, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-9b8659e2671b", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/robotoff.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1728, + "latency_ms": 1229.401, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-a580a089b8cc", + "path_leak": true, + "required_files": [ + "migrations/0007_review_queue_indexes.sql", + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/gtin.ts", + "test/domain.test.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "migrations/0007_review_queue_indexes.sql" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.8333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.8333, + "recall_at_16000_tokens": 0.8333, + "first_hit_rank": 1, + "tokens_delivered": 1676, + "latency_ms": 1219.479, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-b603f405353b", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "scripts/adapters/robotoff-api.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 5, + "tokens_delivered": 1694, + "latency_ms": 1402.49, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-b7e0001c5df7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [ + "scripts/guarded-publication.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1695, + "latency_ms": 2195.973, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-be2529dd2357", + "path_leak": true, + "required_files": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "found": [ + "scripts/guarded-publication.ts", + "scripts/reconcile.ts", + "scripts/review-bundles.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1689, + "latency_ms": 3575.588, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-c07e964dd0d2", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "scripts/adapters/open-food-facts.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1706, + "latency_ms": 1249.709, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-c1ab8fe455d4", + "path_leak": true, + "required_files": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/label-image.ts", + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff-ingredients-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1709, + "latency_ms": 2553.103, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-c634f70476c4", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 15, + "tokens_delivered": 1711, + "latency_ms": 2559.364, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-c725a3c50f27", + "path_leak": false, + "required_files": [ + "src/App.tsx" + ], + "found": [], + "missed": [ + "src/App.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2509, + "latency_ms": 2564.055, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-cadcf1bd67cf", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [], + "missed": [ + "test/ingestion.test.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1052, + "latency_ms": 2292.836, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-cb4efe626dc5", + "path_leak": false, + "required_files": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/review-bundles.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1689, + "latency_ms": 1531.797, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-d131feb0b34a", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [ + "scripts/guarded-publication.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 225, + "latency_ms": 2872.903, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-d2003fb0158f", + "path_leak": true, + "required_files": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "found": [], + "missed": [ + "test/api.worker.test.ts", + "worker/coverage.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1666, + "latency_ms": 1421.432, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-d3b0639e3065", + "path_leak": false, + "required_files": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json", + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/review-54faa8d0bdd98b530bb8/checksums.sha256", + "review-decisions/review-54faa8d0bdd98b530bb8/manifest.json" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 1699, + "latency_ms": 1606.215, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-d3e9f665abd0", + "path_leak": false, + "required_files": [ + "shared/nutrition.ts", + "test/api.worker.test.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "found": [ + "shared/nutrition.ts", + "test/domain.test.ts", + "worker/catalog.ts" + ], + "missed": [ + "test/api.worker.test.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.2727, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 2, + "tokens_delivered": 4653, + "latency_ms": 1109.369, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-df42c030ac4d", + "path_leak": false, + "required_files": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/robotoff-api.ts", + "scripts/adapters/robotoff.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1702, + "latency_ms": 1548.811, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-e9b06828e033", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts-api.ts", + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 15, + "tokens_delivered": 1694, + "latency_ms": 1023.378, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-e9dff924a117", + "path_leak": false, + "required_files": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "found": [ + "scripts/adapters/open-food-facts.ts", + "scripts/reconcile.ts", + "shared/ingredients.ts", + "test/domain.test.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 1685, + "latency_ms": 1768.631, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-ee7d155b7b9e", + "path_leak": false, + "required_files": [ + "config/official-brand-sources.json" + ], + "found": [], + "missed": [ + "config/official-brand-sources.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1680, + "latency_ms": 2845.032, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-f4da150e6ba9", + "path_leak": true, + "required_files": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts", + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "found": [ + "scripts/decision-drift-audit.ts", + "test/ingestion.test.ts" + ], + "missed": [ + "review-decisions/active-bundles.json", + "scripts/adapters/robotoff-ingredients-api.ts", + "scripts/audit-decisions.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.4, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 5, + "tokens_delivered": 1694, + "latency_ms": 2409.048, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-f9a264cd2de7", + "path_leak": false, + "required_files": [ + "scripts/guarded-publication.ts" + ], + "found": [ + "scripts/guarded-publication.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 10692, + "latency_ms": 2231.152, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "protein-index-fda927717fdd", + "path_leak": false, + "required_files": [ + "test/ingestion.test.ts" + ], + "found": [ + "test/ingestion.test.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1706, + "latency_ms": 1492.336, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/V-site-health.json b/benchmarks/context-retrieval/results/V-site-health.json new file mode 100644 index 00000000..b172c2db --- /dev/null +++ b/benchmarks/context-retrieval/results/V-site-health.json @@ -0,0 +1,8095 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "site-health", + "head": "c4ea582e0a8371c566e3e832485fc33da1a318c6" + }, + "tier": "small", + "tier_code_files": 69, + "corpus_counts": { + "cases": 88, + "path_leak": 45, + "baseline_blind": 43, + "multi_file": 49 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.2311, + "control": 0.0114, + "reason": null + }, + "needs_raw_payload_check": [ + { + "provider_id": "codevetter-structural-context", + "why": "scored exactly zero — read the raw payload before believing it" + } + ], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "complete mobile preview pane platform split", + "revision": "cde2e33f06097c1ee56dd6b7f61f9ea135b7b34b", + "returned": [ + "apps/mobile/src/lib/use-voice-input.ts", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/src/components/preview-pane.ts", + "packages/bridge/tsconfig.json", + "apps/mobile/src/components/ui.tsx" + ] + }, + { + "provider_id": "random-files", + "query": "omit unavailable public product links", + "revision": "7d2d155a89368326a4346e874f9b9a866e922151", + "returned": [ + "openspec/changes/archive/2026-07-13-fleet-marketing-control-plane/.openspec.yaml", + "fleet-ops/services/content-factory/scripts/render-content-package.js", + "fleet-ops/services/reel-pipeline/test/fixtures/saas-maker-improvement.json", + "fleet-ops/services/drank/data/fleet-dr.json", + "fleet-ops/apps/mobile-cockpit/openspec/config.yaml" + ] + }, + { + "provider_id": "random-files", + "query": "make native simulator workflow functional", + "revision": "eff69187f6b7d044a8ae946b8ffddc60b1542eff", + "returned": [ + "packages/bridge/tsconfig.build.json", + "apps/mobile/test/layout.test.ts", + "apps/mobile/src/app/index.tsx", + "apps/mobile/test/ios-scene-lifecycle-plugin.test.js", + "apps/mobile/modules/cockpit-voice/expo-module.config.json" + ] + }, + { + "provider_id": "churn-ranked", + "query": "show Search reporting period", + "revision": "4b357538b4bd04e389d89b138127dd036152c466", + "returned": [ + "package.json", + "foundry/ops/config/projects.json", + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/test/marketing-program.test.mjs" + ] + }, + { + "provider_id": "churn-ranked", + "query": "expose provider console links", + "revision": "66d72c5c3b4060fbd32304aad8d1023fd704deb8", + "returned": [ + "package.json", + "foundry/ops/config/projects.json", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json" + ] + }, + { + "provider_id": "graphify", + "query": "show Search reporting period", + "revision": "4b357538b4bd04e389d89b138127dd036152c466", + "returned": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/skills/fleet-deploy-parity/SKILL.md", + "foundry/ops/scripts/agent-bin/fleet-skill-run.mjs", + "foundry/ops/skills/cloudflare-spend-guard/scripts/record-spend-snapshot.mjs", + "foundry/marketing/reel-pipeline/editorial/src/mashup/retrieve.py" + ] + }, + { + "provider_id": "graphify", + "query": "align fleet public sync and live audit", + "revision": "06bbacd1637eda6ed665137b6618ee5966edb7fd", + "returned": [ + "fleet-ops/docs/agent-indexing-standard.md", + "fleet-ops/public/README.md", + "fleet-ops/docs/fleet-sync.md", + "fleet-ops/apps/ops-console/src/lib/fleet-data.ts", + "fleet-ops/services/reel-pipeline/config/live-generation-readiness.json" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 88, + "mean_recall_at_5": 0.0114, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0152, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0492, + "full_recall_at_20": 0.0227, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0114, + "mean_recall_at_16000_tokens": 0.0331, + "mean_precision_at_10": 0.0034, + "zero_hit_rate": 0.8977, + "median_tokens_delivered": 28531, + "median_latency_ms": 14.3585, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 88, + "mean_recall_at_5": 0.0114, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0152, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0492, + "full_recall_at_20": 0.0227, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0114, + "mean_recall_at_16000_tokens": 0.0331, + "mean_precision_at_10": 0.0034, + "zero_hit_rate": 0.8977, + "median_tokens_delivered": 28531, + "median_latency_ms": 14.3585, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 45, + "mean_recall_at_5": 0.0111, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0185, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0556, + "full_recall_at_20": 0.0222, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0111, + "mean_recall_at_16000_tokens": 0.0241, + "mean_precision_at_10": 0.0044, + "zero_hit_rate": 0.8667, + "median_tokens_delivered": 33112, + "median_latency_ms": 14.336, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 43, + "mean_recall_at_5": 0.0116, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0116, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0426, + "full_recall_at_20": 0.0233, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0116, + "mean_recall_at_16000_tokens": 0.0426, + "mean_precision_at_10": 0.0023, + "zero_hit_rate": 0.9302, + "median_tokens_delivered": 27834, + "median_latency_ms": 14.445, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 88 + }, + "resources": { + "baseline_rss_mb": 27088, + "peak_rss_mb": 27282.7, + "delta_rss_mb": 194.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 88, + "mean_recall_at_5": 0.1315, + "full_recall_at_5": 0.0795, + "mean_recall_at_10": 0.1947, + "full_recall_at_10": 0.1136, + "mean_recall_at_20": 0.2466, + "full_recall_at_20": 0.1705, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0038, + "mean_recall_at_16000_tokens": 0.068, + "mean_precision_at_10": 0.0466, + "zero_hit_rate": 0.6591, + "median_tokens_delivered": 139176, + "median_latency_ms": 76.6615, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 88, + "mean_recall_at_5": 0.1315, + "full_recall_at_5": 0.0795, + "mean_recall_at_10": 0.1947, + "full_recall_at_10": 0.1136, + "mean_recall_at_20": 0.2466, + "full_recall_at_20": 0.1705, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0038, + "mean_recall_at_16000_tokens": 0.068, + "mean_precision_at_10": 0.0466, + "zero_hit_rate": 0.6591, + "median_tokens_delivered": 139176, + "median_latency_ms": 76.6615, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 45, + "mean_recall_at_5": 0.0756, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.133, + "full_recall_at_10": 0.0222, + "mean_recall_at_20": 0.1938, + "full_recall_at_20": 0.0889, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0074, + "mean_recall_at_16000_tokens": 0.0367, + "mean_precision_at_10": 0.0489, + "zero_hit_rate": 0.6667, + "median_tokens_delivered": 68366, + "median_latency_ms": 65.669, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 43, + "mean_recall_at_5": 0.1899, + "full_recall_at_5": 0.1628, + "mean_recall_at_10": 0.2593, + "full_recall_at_10": 0.2093, + "mean_recall_at_20": 0.3019, + "full_recall_at_20": 0.2558, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1008, + "mean_precision_at_10": 0.0442, + "zero_hit_rate": 0.6512, + "median_tokens_delivered": 142313, + "median_latency_ms": 81.148, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 88 + }, + "resources": { + "baseline_rss_mb": 27250.5, + "peak_rss_mb": 28675.8, + "delta_rss_mb": 1425.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 6, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 369.5, + "median_latency_ms": 1605.0825, + "payload_kind": [ + "excerpts" + ], + "unavailable": 3 + }, + "baseline_missed": { + "cases": 6, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 369.5, + "median_latency_ms": 1605.0825, + "payload_kind": [ + "excerpts" + ], + "unavailable": 3 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 1, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 3162.711, + "payload_kind": [ + "excerpts" + ], + "unavailable": 1 + }, + "no_path_leak": { + "cases": 5, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 739, + "median_latency_ms": 1452.995, + "payload_kind": [ + "excerpts" + ], + "unavailable": 2 + } + }, + "outcomes": { + "answered": 3, + "refused-own-capacity-limit": 3 + }, + "abandoned": { + "after_cases": 6, + "remaining": 82, + "reason": "3 consecutive index failures; the arm cannot index this repository" + }, + "resources": { + "baseline_rss_mb": 28021.5, + "peak_rss_mb": 28774.7, + "delta_rss_mb": 753.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 284930048 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 88, + "mean_recall_at_5": 0.1677, + "full_recall_at_5": 0.1136, + "mean_recall_at_10": 0.2265, + "full_recall_at_10": 0.1705, + "mean_recall_at_20": 0.2368, + "full_recall_at_20": 0.1705, + "mean_recall_at_1000_tokens": 0.1659, + "mean_recall_at_4000_tokens": 0.2311, + "mean_recall_at_16000_tokens": 0.2368, + "mean_precision_at_10": 0.0455, + "zero_hit_rate": 0.6364, + "median_tokens_delivered": 1684, + "median_latency_ms": 11051.3825, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 88, + "mean_recall_at_5": 0.1677, + "full_recall_at_5": 0.1136, + "mean_recall_at_10": 0.2265, + "full_recall_at_10": 0.1705, + "mean_recall_at_20": 0.2368, + "full_recall_at_20": 0.1705, + "mean_recall_at_1000_tokens": 0.1659, + "mean_recall_at_4000_tokens": 0.2311, + "mean_recall_at_16000_tokens": 0.2368, + "mean_precision_at_10": 0.0455, + "zero_hit_rate": 0.6364, + "median_tokens_delivered": 1684, + "median_latency_ms": 11051.3825, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 45, + "mean_recall_at_5": 0.2243, + "full_recall_at_5": 0.1333, + "mean_recall_at_10": 0.2838, + "full_recall_at_10": 0.2, + "mean_recall_at_20": 0.2926, + "full_recall_at_20": 0.2, + "mean_recall_at_1000_tokens": 0.1875, + "mean_recall_at_4000_tokens": 0.2815, + "mean_recall_at_16000_tokens": 0.2926, + "mean_precision_at_10": 0.0559, + "zero_hit_rate": 0.5111, + "median_tokens_delivered": 1682, + "median_latency_ms": 10080.05, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 43, + "mean_recall_at_5": 0.1085, + "full_recall_at_5": 0.093, + "mean_recall_at_10": 0.1667, + "full_recall_at_10": 0.1395, + "mean_recall_at_20": 0.1783, + "full_recall_at_20": 0.1395, + "mean_recall_at_1000_tokens": 0.1434, + "mean_recall_at_4000_tokens": 0.1783, + "mean_recall_at_16000_tokens": 0.1783, + "mean_precision_at_10": 0.0347, + "zero_hit_rate": 0.7674, + "median_tokens_delivered": 1687, + "median_latency_ms": 13474.264, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 88 + }, + "resources": { + "baseline_rss_mb": 27239.6, + "peak_rss_mb": 43815.1, + "delta_rss_mb": 16575.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 1220976640 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "site-health-00d7bb046471", + "path_leak": false, + "required_files": [ + "apps/backend/config/projects.json" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23275, + "latency_ms": 38.808, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-010e4aebe75e", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 669466, + "latency_ms": 25.27, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-054409454be6", + "path_leak": false, + "required_files": [ + ".gitmodules" + ], + "found": [], + "missed": [ + ".gitmodules" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44685, + "latency_ms": 15.985, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-071417754558", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/next.config.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16899, + "latency_ms": 22.056, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-07ad25caddd4", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22403, + "latency_ms": 15.809, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-0ba9b5f5948b", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27851, + "latency_ms": 14.445, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-0baeb9010cb3", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62897, + "latency_ms": 15.189, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-132847b20d6d", + "path_leak": false, + "required_files": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17187, + "latency_ms": 16.636, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-18e656d8741b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 74067, + "latency_ms": 13.983, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-19f05b318577", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18822, + "latency_ms": 13.234, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-1ca619fba9f2", + "path_leak": true, + "required_files": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 118893, + "latency_ms": 14.381, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-1d00d5a8b1af", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27834, + "latency_ms": 13.714, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-21839f193280", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [], + "missed": [ + "app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12713, + "latency_ms": 11.879, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-2451fc913f88", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34583, + "latency_ms": 14.206, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-2991f2b749b6", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28233, + "latency_ms": 14.2, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-2d81c1441e17", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20297, + "latency_ms": 18.776, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-2e7392f7dccd", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28206, + "latency_ms": 14.961, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-3224f6662580", + "path_leak": true, + "required_files": [ + "apps/mobile/src/components/preview-pane.d.ts", + "apps/mobile/src/components/preview-pane.ts" + ], + "found": [ + "apps/mobile/src/components/preview-pane.ts" + ], + "missed": [ + "apps/mobile/src/components/preview-pane.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 20739, + "latency_ms": 11.968, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-32e6fd1b5c87", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27802, + "latency_ms": 15.552, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-40426bd7a03a", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27089, + "latency_ms": 15.981, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-41ad25bba286", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39614, + "latency_ms": 15.496, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-464fe2d9d985", + "path_leak": false, + "required_files": [ + "biome.json" + ], + "found": [], + "missed": [ + "biome.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 97735, + "latency_ms": 10.723, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-4b3332684075", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28496, + "latency_ms": 14.004, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-4c00f66be11b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20816, + "latency_ms": 15.663, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-4efb5e0e9f6c", + "path_leak": false, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36743, + "latency_ms": 11.37, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-4f8e172c77c0", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23813, + "latency_ms": 11.937, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-58ce619558e4", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84109, + "latency_ms": 14.697, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5947cb9235a4", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13196, + "latency_ms": 13.101, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5add561d3a1c", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31271, + "latency_ms": 16.027, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5c274bfb8d8c", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47320, + "latency_ms": 11.952, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5c737aa349c4", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [ + "app/layout.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 13140, + "latency_ms": 11.716, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5e9dd3d3a183", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35626, + "latency_ms": 13.234, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5f057c9840e6", + "path_leak": true, + "required_files": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28196, + "latency_ms": 13.086, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-5ff8c1d23d1a", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27122, + "latency_ms": 12.736, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-607ffead8d59", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40077, + "latency_ms": 15.335, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-6410c21e39fb", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41266, + "latency_ms": 14.078, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-6806c601cd49", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 76806, + "latency_ms": 12.572, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-6a61e9774eee", + "path_leak": true, + "required_files": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/design-workflow.mjs" + ], + "found": [ + "foundry/ops/scripts/design-workflow.mjs" + ], + "missed": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 13, + "tokens_delivered": 21229, + "latency_ms": 13.674, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-6e4082265380", + "path_leak": true, + "required_files": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28566, + "latency_ms": 14.895, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-70a328abccdb", + "path_leak": false, + "required_files": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "found": [], + "missed": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15368, + "latency_ms": 16.557, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-747d98b7face", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 61753, + "latency_ms": 13.293, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-756936a2820c", + "path_leak": false, + "required_files": [ + "scripts/prepare-node-pty.mjs" + ], + "found": [], + "missed": [ + "scripts/prepare-node-pty.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16437, + "latency_ms": 12.826, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-834566d04be7", + "path_leak": true, + "required_files": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "found": [ + "foundry/services/reel-pipeline/src/film-skills.js" + ], + "missed": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 42577, + "latency_ms": 15.053, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-864bb66e08c9", + "path_leak": false, + "required_files": [ + "foundry/ops/config/projects.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 64399, + "latency_ms": 14.912, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-8c06ab66e57f", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22437, + "latency_ms": 14.196, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-90d2bee838d8", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33970, + "latency_ms": 15.311, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-93e088e6145f", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29219, + "latency_ms": 14.749, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-98062c56ade9", + "path_leak": true, + "required_files": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/project-catalog.test.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/project-catalog.test.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21450, + "latency_ms": 16.85, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-9afe0805eedb", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "found": [ + "foundry/packages/portfolio-project-strip/src/catalog.ts" + ], + "missed": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 33719, + "latency_ms": 15.708, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-9d823e16ea99", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/search-console.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/search-console.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 79649, + "latency_ms": 14.336, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-9e0367722215", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28146, + "latency_ms": 13.733, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-9f748e8bfd7a", + "path_leak": false, + "required_files": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts", + "packages/protocol/src/index.ts" + ], + "found": [ + "packages/protocol/src/index.ts" + ], + "missed": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 17, + "tokens_delivered": 17551, + "latency_ms": 10.822, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-a29095247805", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22657, + "latency_ms": 14.606, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-a67cc73f8c17", + "path_leak": true, + "required_files": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "found": [ + "fleet-ops/public/products.json" + ], + "missed": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 9, + "tokens_delivered": 20530, + "latency_ms": 12.724, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-a766a9fe6db6", + "path_leak": true, + "required_files": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "found": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 56960, + "latency_ms": 11.277, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-a7e020f5380c", + "path_leak": true, + "required_files": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 87173, + "latency_ms": 15.687, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-aa702f549db8", + "path_leak": true, + "required_files": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44921, + "latency_ms": 22.256, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-abf1e1c3dcf9", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93641, + "latency_ms": 15.74, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-ac2d8a08ccf7", + "path_leak": true, + "required_files": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25118, + "latency_ms": 15.162, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-ad8c488454cf", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/git-health.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/git-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32389, + "latency_ms": 13.266, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-b311494c1853", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34931, + "latency_ms": 19.036, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-b43186e6c2d6", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22315, + "latency_ms": 14.998, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-b460bbee477e", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/page.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/page.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 119820, + "latency_ms": 15.06, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-b963e53c1599", + "path_leak": false, + "required_files": [ + "foundry/ops/config/marketing-program.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/marketing-program.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22797, + "latency_ms": 15.157, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-b9a9b7fb722e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 108763, + "latency_ms": 15.199, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c055fb854a9c", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30638, + "latency_ms": 17.167, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c05c3a7fd02e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34548, + "latency_ms": 13.78, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c30d08ebb49f", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30735, + "latency_ms": 13.15, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c3cb6dd9d680", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13152, + "latency_ms": 14.17, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c4f7e2d22b0c", + "path_leak": true, + "required_files": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24733, + "latency_ms": 13.725, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c50c5280c323", + "path_leak": true, + "required_files": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12255, + "latency_ms": 15.171, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-c99064f83b83", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 95525, + "latency_ms": 15.04, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-ca50c4d300dd", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33858, + "latency_ms": 13.45, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-d906a7364744", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33529, + "latency_ms": 15.49, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-e132f96c25ac", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15978, + "latency_ms": 13.63, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-e261ca6aa79e", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24720, + "latency_ms": 12.645, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-e288e502d891", + "path_leak": false, + "required_files": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "found": [ + "apps/mobile/src/app/index.tsx" + ], + "missed": [ + "packages/bridge/src/discover.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 14889, + "latency_ms": 11.894, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-e4a52d0adfb6", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 94875, + "latency_ms": 17.158, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-e7dec365bcf9", + "path_leak": false, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26039, + "latency_ms": 14.461, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-e8113343338f", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 88690, + "latency_ms": 14.602, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-eaaf7a67fc35", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42075, + "latency_ms": 13.225, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-efdb446d024f", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21596, + "latency_ms": 13.859, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-f157706d6ddc", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 65479, + "latency_ms": 15.386, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-f27818367be8", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25830, + "latency_ms": 13.983, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-f2b275130ace", + "path_leak": true, + "required_files": [ + "fleet-ops/config/marketing-program.json" + ], + "found": [], + "missed": [ + "fleet-ops/config/marketing-program.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21528, + "latency_ms": 11.188, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-f7557d39e6e6", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 98211, + "latency_ms": 13.772, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-f8b07be6f191", + "path_leak": true, + "required_files": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "found": [], + "missed": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13658, + "latency_ms": 10.941, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "site-health-fa0e527907ad", + "path_leak": true, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33112, + "latency_ms": 12.464, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-00d7bb046471", + "path_leak": false, + "required_files": [ + "apps/backend/config/projects.json" + ], + "found": [ + "apps/backend/config/projects.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 87102, + "latency_ms": 187.15, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-010e4aebe75e", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 184580, + "latency_ms": 69.076, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-054409454be6", + "path_leak": false, + "required_files": [ + ".gitmodules" + ], + "found": [], + "missed": [ + ".gitmodules" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 52509, + "latency_ms": 23.254, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-071417754558", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/next.config.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159680, + "latency_ms": 91.68, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-07ad25caddd4", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 57622, + "latency_ms": 61.125, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-0ba9b5f5948b", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159689, + "latency_ms": 80.826, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-0baeb9010cb3", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 141878, + "latency_ms": 93.756, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-132847b20d6d", + "path_leak": false, + "required_files": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "found": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 160128, + "latency_ms": 82.481, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-18e656d8741b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14900, + "latency_ms": 64.01, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-19f05b318577", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "found": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 57622, + "latency_ms": 59.532, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-1ca619fba9f2", + "path_leak": true, + "required_files": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 160932, + "latency_ms": 95.551, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-1d00d5a8b1af", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159689, + "latency_ms": 79.554, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-21839f193280", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [ + "app/layout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 34162, + "latency_ms": 32.624, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-2451fc913f88", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 131913, + "latency_ms": 69.608, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-2991f2b749b6", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 57622, + "latency_ms": 70.369, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-2d81c1441e17", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 191617, + "latency_ms": 145.183, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-2e7392f7dccd", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32357, + "latency_ms": 63.035, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-3224f6662580", + "path_leak": true, + "required_files": [ + "apps/mobile/src/components/preview-pane.d.ts", + "apps/mobile/src/components/preview-pane.ts" + ], + "found": [], + "missed": [ + "apps/mobile/src/components/preview-pane.d.ts", + "apps/mobile/src/components/preview-pane.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123413, + "latency_ms": 18.666, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-32e6fd1b5c87", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37794, + "latency_ms": 32.01, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-40426bd7a03a", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 144567, + "latency_ms": 76.582, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-41ad25bba286", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 179520, + "latency_ms": 118.735, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-464fe2d9d985", + "path_leak": false, + "required_files": [ + "biome.json" + ], + "found": [ + "biome.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 38099, + "latency_ms": 25.593, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-4b3332684075", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 143425, + "latency_ms": 90.439, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-4c00f66be11b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 147821, + "latency_ms": 91.054, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-4efb5e0e9f6c", + "path_leak": false, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38341, + "latency_ms": 33.604, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-4f8e172c77c0", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18532, + "latency_ms": 64.809, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-58ce619558e4", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 181851, + "latency_ms": 86.13, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5947cb9235a4", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32357, + "latency_ms": 56.501, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5add561d3a1c", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "found": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "missed": [], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.8, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 184098, + "latency_ms": 122.21, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5c274bfb8d8c", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68366, + "latency_ms": 28.762, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5c737aa349c4", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [ + "app/layout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 34196, + "latency_ms": 22.291, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5e9dd3d3a183", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 39535, + "latency_ms": 24.983, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5f057c9840e6", + "path_leak": true, + "required_files": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [ + "foundry/ops/config/projects.json" + ], + "missed": [ + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.1429, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 3, + "tokens_delivered": 32020, + "latency_ms": 65.669, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-5ff8c1d23d1a", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 57549, + "latency_ms": 55.498, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-607ffead8d59", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 181851, + "latency_ms": 80.968, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-6410c21e39fb", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 180464, + "latency_ms": 89.013, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-6806c601cd49", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 91941, + "latency_ms": 22.666, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-6a61e9774eee", + "path_leak": true, + "required_files": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/design-workflow.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/design-workflow.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46309, + "latency_ms": 63.837, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-6e4082265380", + "path_leak": true, + "required_files": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43420, + "latency_ms": 62.17, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-70a328abccdb", + "path_leak": false, + "required_files": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "found": [], + "missed": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42408, + "latency_ms": 64.847, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-747d98b7face", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18532, + "latency_ms": 60.26, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-756936a2820c", + "path_leak": false, + "required_files": [ + "scripts/prepare-node-pty.mjs" + ], + "found": [], + "missed": [ + "scripts/prepare-node-pty.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 119636, + "latency_ms": 20.189, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-834566d04be7", + "path_leak": true, + "required_files": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "found": [], + "missed": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44808, + "latency_ms": 62.831, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-864bb66e08c9", + "path_leak": false, + "required_files": [ + "foundry/ops/config/projects.json" + ], + "found": [ + "foundry/ops/config/projects.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 179290, + "latency_ms": 86.756, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-8c06ab66e57f", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 147535, + "latency_ms": 81.148, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-90d2bee838d8", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "missed": [ + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 142904, + "latency_ms": 81.282, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-93e088e6145f", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43420, + "latency_ms": 62.995, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-98062c56ade9", + "path_leak": true, + "required_files": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/project-catalog.test.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "found": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "missed": [ + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 191617, + "latency_ms": 87.179, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-9afe0805eedb", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 179520, + "latency_ms": 83.162, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-9d823e16ea99", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/search-console.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/search-console.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 147424, + "latency_ms": 74.035, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-9e0367722215", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 142313, + "latency_ms": 90.088, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-9f748e8bfd7a", + "path_leak": false, + "required_files": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts", + "packages/protocol/src/index.ts" + ], + "found": [], + "missed": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts", + "packages/protocol/src/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123413, + "latency_ms": 22.683, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-a29095247805", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42408, + "latency_ms": 57.031, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-a67cc73f8c17", + "path_leak": true, + "required_files": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "found": [], + "missed": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 56617, + "latency_ms": 34.728, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-a766a9fe6db6", + "path_leak": true, + "required_files": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "found": [], + "missed": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34699, + "latency_ms": 27.264, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-a7e020f5380c", + "path_leak": true, + "required_files": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "found": [ + "fleet-ops/config/projects.json" + ], + "missed": [ + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.1429, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 5, + "tokens_delivered": 62450, + "latency_ms": 31.153, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-aa702f549db8", + "path_leak": true, + "required_files": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "found": [ + "apps/backend/config/projects.json" + ], + "missed": [ + "apps/backend/test/catalog-boundary.test.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71914, + "latency_ms": 189.698, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-abf1e1c3dcf9", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 138229, + "latency_ms": 67.664, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-ac2d8a08ccf7", + "path_leak": true, + "required_files": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "found": [], + "missed": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67468, + "latency_ms": 22.454, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-ad8c488454cf", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/git-health.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/git-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68366, + "latency_ms": 32.912, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-b311494c1853", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "missed": [ + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 141888, + "latency_ms": 87.676, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-b43186e6c2d6", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 142313, + "latency_ms": 91.893, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-b460bbee477e", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/page.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/page.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159658, + "latency_ms": 78.123, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-b963e53c1599", + "path_leak": false, + "required_files": [ + "foundry/ops/config/marketing-program.json" + ], + "found": [ + "foundry/ops/config/marketing-program.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 147535, + "latency_ms": 85.519, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-b9a9b7fb722e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 191618, + "latency_ms": 85.148, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c055fb854a9c", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 141888, + "latency_ms": 88.069, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c05c3a7fd02e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 145710, + "latency_ms": 79.652, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c30d08ebb49f", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67192, + "latency_ms": 75.44, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c3cb6dd9d680", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 143521, + "latency_ms": 82.863, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c4f7e2d22b0c", + "path_leak": true, + "required_files": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159658, + "latency_ms": 76.761, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c50c5280c323", + "path_leak": true, + "required_files": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 161810, + "latency_ms": 91.285, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-c99064f83b83", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159680, + "latency_ms": 81.867, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-ca50c4d300dd", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "missed": [ + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 144090, + "latency_ms": 87.784, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-d906a7364744", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 184565, + "latency_ms": 76.741, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-e132f96c25ac", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159680, + "latency_ms": 90.847, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-e261ca6aa79e", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10973, + "latency_ms": 58.894, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-e288e502d891", + "path_leak": false, + "required_files": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "found": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 123181, + "latency_ms": 18.744, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-e4a52d0adfb6", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159680, + "latency_ms": 89.963, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-e7dec365bcf9", + "path_leak": false, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140123, + "latency_ms": 95.441, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-e8113343338f", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 184580, + "latency_ms": 94.367, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-eaaf7a67fc35", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "found": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json" + ], + "missed": [ + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 61019, + "latency_ms": 67.872, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-efdb446d024f", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [ + "foundry/ops/test/public-products.test.mjs" + ], + "missed": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 145934, + "latency_ms": 185.049, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-f157706d6ddc", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "missed": [ + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 146213, + "latency_ms": 157.269, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-f27818367be8", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 159680, + "latency_ms": 112.457, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-f2b275130ace", + "path_leak": true, + "required_files": [ + "fleet-ops/config/marketing-program.json" + ], + "found": [ + "fleet-ops/config/marketing-program.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 67468, + "latency_ms": 41.969, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-f7557d39e6e6", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [ + "foundry/ops/test/public-products.test.mjs" + ], + "missed": [ + "foundry/apps/public/public-directory/public/_redirects" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 146153, + "latency_ms": 165.587, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-f8b07be6f191", + "path_leak": true, + "required_files": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "found": [ + "apps/mobile/app.json" + ], + "missed": [ + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 38020, + "latency_ms": 65.609, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "site-health-fa0e527907ad", + "path_leak": true, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67192, + "latency_ms": 108.32, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "site-health-00d7bb046471", + "path_leak": false, + "required_files": [ + "apps/backend/config/projects.json" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1295, + "latency_ms": 516.954, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "site-health-010e4aebe75e", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 739, + "latency_ms": 1452.995, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "site-health-054409454be6", + "path_leak": false, + "required_files": [ + ".gitmodules" + ], + "found": [], + "missed": [ + ".gitmodules" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1260, + "latency_ms": 438.925, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "site-health-071417754558", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/next.config.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 3162.711, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "site-health-07ad25caddd4", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1757.17, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "site-health-0ba9b5f5948b", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2880.227, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-00d7bb046471", + "path_leak": false, + "required_files": [ + "apps/backend/config/projects.json" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1669, + "latency_ms": 1839.153, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-010e4aebe75e", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2863, + "latency_ms": 10685.065, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-054409454be6", + "path_leak": false, + "required_files": [ + ".gitmodules" + ], + "found": [], + "missed": [ + ".gitmodules" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1698, + "latency_ms": 4483.493, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-071417754558", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/next.config.ts" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/next.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1662, + "latency_ms": 32339.365, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-07ad25caddd4", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4700, + "latency_ms": 14708.721, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-0ba9b5f5948b", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3767, + "latency_ms": 17818.032, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-0baeb9010cb3", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 271, + "latency_ms": 16000.481, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-132847b20d6d", + "path_leak": false, + "required_files": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/automation-registry.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1667, + "latency_ms": 19590.889, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-18e656d8741b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "found": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1667, + "latency_ms": 8595.653, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-19f05b318577", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "found": [ + "foundry/ops/lib/fleet-automation/registry.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 1671, + "latency_ms": 10992.955, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-1ca619fba9f2", + "path_leak": true, + "required_files": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "found": [ + "foundry/helpers/chatgpt-connections/src/activation.ts", + "foundry/helpers/chatgpt-connections/src/verify-activation.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5667, + "latency_ms": 16812.049, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-1d00d5a8b1af", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/wrangler.jsonc" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1668, + "latency_ms": 15728.903, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-21839f193280", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [ + "app/layout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5350, + "latency_ms": 1043.437, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-2451fc913f88", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/pages/search.astro", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 1670, + "latency_ms": 11715.931, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-2991f2b749b6", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/.fleet/design-review.json", + "foundry/apps/public-directory/src/components/Fleet.astro", + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3754, + "latency_ms": 533.961, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-2d81c1441e17", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "found": [ + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/posthog-outcomes-collect.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1668, + "latency_ms": 17902.084, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-2e7392f7dccd", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1688, + "latency_ms": 7388.121, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-3224f6662580", + "path_leak": true, + "required_files": [ + "apps/mobile/src/components/preview-pane.d.ts", + "apps/mobile/src/components/preview-pane.ts" + ], + "found": [ + "apps/mobile/src/components/preview-pane.ts" + ], + "missed": [ + "apps/mobile/src/components/preview-pane.d.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 7946, + "latency_ms": 881.514, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-32e6fd1b5c87", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1674, + "latency_ms": 1561.265, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-40426bd7a03a", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/layouts/Layout.astro", + "foundry/ops/config/root-brands.json", + "foundry/ops/lib/root-brand-contract.mjs", + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1665, + "latency_ms": 13770.662, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-41ad25bba286", + "path_leak": true, + "required_files": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1670, + "latency_ms": 17616.848, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-464fe2d9d985", + "path_leak": false, + "required_files": [ + "biome.json" + ], + "found": [], + "missed": [ + "biome.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1457, + "latency_ms": 641.313, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-4b3332684075", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1687, + "latency_ms": 13731.897, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-4c00f66be11b", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/agent-bin/ops-console", + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "found": [ + "foundry/ops/scripts/agent-bin/ops-console" + ], + "missed": [ + "foundry/ops/test/ops-console-runtime.test.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1664, + "latency_ms": 14522.026, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-4efb5e0e9f6c", + "path_leak": false, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3891, + "latency_ms": 1477.349, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-4f8e172c77c0", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1684, + "latency_ms": 6489.652, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-58ce619558e4", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [ + "foundry/ops/lib/posthog-outcomes.mjs" + ], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 15, + "tokens_delivered": 1698, + "latency_ms": 17048.17, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-5947cb9235a4", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3648, + "latency_ms": 7329.508, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-5add561d3a1c", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/geo-observatory.json", + "foundry/ops/config/marketing-program.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/project-catalog.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1662, + "latency_ms": 5101.3, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-5c274bfb8d8c", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-init.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1673, + "latency_ms": 2425.656, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-5c737aa349c4", + "path_leak": false, + "required_files": [ + "app/layout.tsx" + ], + "found": [ + "app/layout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2134, + "latency_ms": 538.505, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-5e9dd3d3a183", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [ + "fleet-ops/scripts/apply-agent-surfaces.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2105, + "latency_ms": 1636.208, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-5f057c9840e6", + "path_leak": true, + "required_files": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/fleet-deploy-guard.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [ + "foundry/ops/scripts/fleet-deploy-guard.sh" + ], + "missed": [ + "foundry/ops/config/projects.json", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh", + "foundry/ops/skills/token-budget/scripts/token-budget.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.1429, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.1429, + "recall_at_4000_tokens": 0.1429, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 3, + "tokens_delivered": 1685, + "latency_ms": 7269.322, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-5ff8c1d23d1a", + "path_leak": false, + "required_files": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "found": [], + "missed": [ + "foundry/apps/public-directory/src/styles/globals.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1692, + "latency_ms": 9061.75, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-607ffead8d59", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "found": [ + "foundry/ops/lib/posthog-outcomes.mjs" + ], + "missed": [ + "foundry/ops/lib/d1-outcomes.mjs", + "foundry/ops/scripts/native-code-health.mjs", + "foundry/ops/test/user-metrics-privacy.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 15, + "tokens_delivered": 1698, + "latency_ms": 973.449, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-6410c21e39fb", + "path_leak": false, + "required_files": [ + "foundry/ops/workflows" + ], + "found": [], + "missed": [ + "foundry/ops/workflows" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1671, + "latency_ms": 7102.356, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-6806c601cd49", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/fleet-deploy-guard.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6184, + "latency_ms": 1459.801, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-6a61e9774eee", + "path_leak": true, + "required_files": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/design-workflow.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/design-workflow.json", + "foundry/ops/lib/design-workflow.mjs", + "foundry/ops/scripts/agent-stack.sh", + "foundry/ops/scripts/design-workflow.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4192, + "latency_ms": 7396.928, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-6e4082265380", + "path_leak": true, + "required_files": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/services/drank/app/layout.tsx", + "foundry/tools/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1712, + "latency_ms": 10105.923, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-70a328abccdb", + "path_leak": false, + "required_files": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "found": [], + "missed": [ + "foundry/apps/setline/tests/rendered-html.test.mjs", + "foundry/apps/setline/worker/schema.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4540, + "latency_ms": 8510.275, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-747d98b7face", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/fleet-health.sh", + "foundry/ops/scripts/link-project-agent-assets.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1660, + "latency_ms": 6566.491, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-756936a2820c", + "path_leak": false, + "required_files": [ + "scripts/prepare-node-pty.mjs" + ], + "found": [], + "missed": [ + "scripts/prepare-node-pty.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4462, + "latency_ms": 939.271, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-834566d04be7", + "path_leak": true, + "required_files": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js", + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "found": [ + "foundry/services/reel-pipeline/src/local-video-forge-coordinator.js", + "foundry/services/reel-pipeline/test/forge-operator-console.test.js" + ], + "missed": [ + "foundry/services/reel-pipeline/scripts/local-video-forge.js", + "foundry/services/reel-pipeline/src/coherent-scene-composition.js", + "foundry/services/reel-pipeline/src/film-skills.js", + "foundry/services/reel-pipeline/src/forge-operator-ui.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 1701, + "latency_ms": 7880.801, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-864bb66e08c9", + "path_leak": false, + "required_files": [ + "foundry/ops/config/projects.json" + ], + "found": [], + "missed": [ + "foundry/ops/config/projects.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1685, + "latency_ms": 17316.4, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-8c06ab66e57f", + "path_leak": false, + "required_files": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/src/studio/ui.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1665, + "latency_ms": 14527.788, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-90d2bee838d8", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-indexing-request-store.mjs", + "foundry/ops/scripts/search-indexing-request-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1671, + "latency_ms": 13593.347, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-93e088e6145f", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "found": [ + "foundry/ops/lib/skill-run-store.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1690, + "latency_ms": 10080.05, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-98062c56ade9", + "path_leak": true, + "required_files": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/founder-control/registry.mjs", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/project-catalog.test.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/registry.mjs" + ], + "missed": [ + "foundry/ops/config/marketing-program.json", + "foundry/ops/lib/platform-matching.mjs", + "foundry/ops/test/project-catalog.test.mjs", + "foundry/ops/test/seo-geo-publishing.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.2, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 14, + "tokens_delivered": 1676, + "latency_ms": 18936.238, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-9afe0805eedb", + "path_leak": true, + "required_files": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/lib/portfolio-projects.ts", + "foundry/packages/portfolio-project-strip/scripts/generate-catalog.mjs", + "foundry/packages/portfolio-project-strip/src/catalog.ts", + "foundry/packages/portfolio-project-strip/src/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1665, + "latency_ms": 17054.962, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-9d823e16ea99", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/search-console.mjs" + ], + "found": [ + "foundry/ops/lib/search-console.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1689, + "latency_ms": 13777.24, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-9e0367722215", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 191, + "latency_ms": 13479.551, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-9f748e8bfd7a", + "path_leak": false, + "required_files": [ + "apps/mobile/src/lib/connection.tsx", + "apps/mobile/src/lib/project-workspace-state.ts", + "packages/protocol/src/index.ts" + ], + "found": [ + "apps/mobile/src/lib/connection.tsx", + "packages/protocol/src/index.ts" + ], + "missed": [ + "apps/mobile/src/lib/project-workspace-state.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 1668, + "latency_ms": 902.487, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-a29095247805", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/apply-agent-surfaces.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1656, + "latency_ms": 8987.109, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-a67cc73f8c17", + "path_leak": true, + "required_files": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "found": [], + "missed": [ + "fleet-ops/config/public-products.json", + "fleet-ops/lib/public-products.mjs", + "fleet-ops/public/products.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4126, + "latency_ms": 5748.061, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-a766a9fe6db6", + "path_leak": true, + "required_files": [ + "app/data/page.tsx", + "lib/api-timing.ts" + ], + "found": [ + "app/data/page.tsx" + ], + "missed": [ + "lib/api-timing.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 7151, + "latency_ms": 539.65, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-a7e020f5380c", + "path_leak": true, + "required_files": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs", + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "found": [ + "fleet-ops/scripts/sync-spotlight-products.mjs" + ], + "missed": [ + "fleet-ops/config/projects.json", + "fleet-ops/config/public-products.json", + "fleet-ops/config/spotlight-products.json", + "fleet-ops/config/spotlight-sync.json", + "fleet-ops/public/products.json", + "fleet-ops/scripts/cloudflare-resilience-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.1429, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.1429, + "recall_at_4000_tokens": 0.1429, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 7, + "tokens_delivered": 1678, + "latency_ms": 6102.724, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-aa702f549db8", + "path_leak": true, + "required_files": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "found": [], + "missed": [ + "apps/backend/config/projects.json", + "apps/backend/test/catalog-boundary.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1662, + "latency_ms": 886.202, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-abf1e1c3dcf9", + "path_leak": true, + "required_files": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/founder-control-service.test.mjs" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1667, + "recall_at_20": 0.2, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 1678, + "latency_ms": 11412.657, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-ac2d8a08ccf7", + "path_leak": true, + "required_files": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "found": [ + "fleet-ops/lib/toolbox-automation/evidence.mjs", + "fleet-ops/test/toolbox-automation.test.mjs" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1682, + "latency_ms": 2369.326, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-ad8c488454cf", + "path_leak": true, + "required_files": [ + "fleet-ops/scripts/git-health.sh" + ], + "found": [], + "missed": [ + "fleet-ops/scripts/git-health.sh" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1694, + "latency_ms": 2376.952, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-b311494c1853", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-console.mjs", + "foundry/ops/lib/visibility-outcome-store.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1651, + "latency_ms": 13745.214, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-b43186e6c2d6", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/globals.css", + "foundry/helpers/drank/app/layout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1658, + "latency_ms": 13434.328, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-b460bbee477e", + "path_leak": false, + "required_files": [ + "foundry/helpers/drank/app/page.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/drank/app/page.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1669, + "latency_ms": 15060.146, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-b963e53c1599", + "path_leak": false, + "required_files": [ + "foundry/ops/config/marketing-program.json" + ], + "found": [ + "foundry/ops/config/marketing-program.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1691, + "latency_ms": 14315.758, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-b9a9b7fb722e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs", + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "found": [ + "foundry/ops/scripts/agent-bin/ops-console-server.mjs" + ], + "missed": [ + "foundry/ops/lib/audience-fit.mjs", + "foundry/ops/lib/posthog-outcomes.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 6, + "tokens_delivered": 3326, + "latency_ms": 17345.609, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-c055fb854a9c", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/visibility-outcome-store.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1676, + "latency_ms": 13474.264, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-c05c3a7fd02e", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "found": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "missed": [ + "foundry/ops/lib/agent-probe-retry.mjs", + "foundry/ops/test/skill-run-cli.test.mjs", + "foundry/ops/test/skill-run-scorecard.test.mjs" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 1676, + "latency_ms": 14081.031, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-c30d08ebb49f", + "path_leak": false, + "required_files": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [ + "foundry/apps/dashboard/fleet-console/.fleet/design-review.json", + "foundry/apps/dashboard/fleet-console/src/scripts/founder-client.ts", + "foundry/ops/lib/founder-control/connections.mjs" + ], + "missed": [ + "foundry/apps/dashboard/fleet-console/src/pages/metrics.astro", + "foundry/apps/dashboard/fleet-console/src/styles/founder.css", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1696, + "latency_ms": 11109.81, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-c3cb6dd9d680", + "path_leak": false, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1653, + "latency_ms": 13879.663, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-c4f7e2d22b0c", + "path_leak": true, + "required_files": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "found": [], + "missed": [ + "foundry/marketing/reel-pipeline/editorial/src/mashup/cli.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/config.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/ingest/loader.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/pipeline.py", + "foundry/marketing/reel-pipeline/editorial/src/mashup/render/cut.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_ingest_loader.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_render_target_format.py", + "foundry/marketing/reel-pipeline/editorial/tests/test_serve.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5409, + "latency_ms": 15772.865, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-c50c5280c323", + "path_leak": true, + "required_files": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/.fleet/design-review.json", + "foundry/helpers/psi-swarm/web/src/components/DisconnectedPanel.tsx", + "foundry/helpers/psi-swarm/web/src/components/RunDashboard.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1686, + "latency_ms": 17978.928, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-c99064f83b83", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1684, + "latency_ms": 15856.133, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-ca50c4d300dd", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/search-change-receipt-store.mjs", + "foundry/ops/scripts/search-change-receipt-record.mjs", + "foundry/ops/test/founder-control-connections.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1679, + "latency_ms": 13626.876, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-d906a7364744", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "found": [ + "foundry/helpers/psi-swarm/cli/package.json", + "foundry/helpers/psi-swarm/package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1688, + "latency_ms": 17318.063, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-e132f96c25ac", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3524, + "latency_ms": 15387.703, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-e261ca6aa79e", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "found": [ + "foundry/ops/scripts/sync-spotlight-products.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3212, + "latency_ms": 6123.937, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-e288e502d891", + "path_leak": false, + "required_files": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "found": [], + "missed": [ + "apps/mobile/src/app/index.tsx", + "packages/bridge/src/discover.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3164, + "latency_ms": 868.676, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-e4a52d0adfb6", + "path_leak": true, + "required_files": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/scripts/deploy-health.sh", + "foundry/ops/test/deploy-tooling.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1647, + "latency_ms": 15297.617, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-e7dec365bcf9", + "path_leak": false, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1675, + "latency_ms": 12079.393, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-e8113343338f", + "path_leak": false, + "required_files": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "found": [ + "foundry/ops/scripts/check-component-roots.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1679, + "latency_ms": 5098.575, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-eaaf7a67fc35", + "path_leak": true, + "required_files": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "found": [], + "missed": [ + "foundry/ops/config/agent-surfaces-registry.json", + "foundry/ops/config/projects.json", + "foundry/ops/test/agent-surface-inventory.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1684, + "latency_ms": 10780.036, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-efdb446d024f", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/src/data/learnings.ts", + "foundry/apps/public/public-directory/src/data/publicRoutes.ts", + "foundry/ops/test/public-products.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1700, + "latency_ms": 13778.807, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-f157706d6ddc", + "path_leak": true, + "required_files": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/scripts/search-console-collect.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "found": [ + "foundry/ops/lib/founder-control/connections.mjs", + "foundry/ops/scripts/search-console-collect.mjs" + ], + "missed": [ + "foundry/ops/lib/visibility-projects.mjs", + "foundry/ops/test/founder-control-connections.test.mjs", + "foundry/ops/test/root-search-query-contract.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 8, + "tokens_delivered": 1686, + "latency_ms": 13783.725, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-f27818367be8", + "path_leak": false, + "required_files": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "found": [], + "missed": [ + "foundry/helpers/psi-swarm/patches/blume@1.0.4.patch", + "foundry/helpers/psi-swarm/pnpm-workspace.yaml", + "foundry/ops/scripts/manual-component-deploy.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3403, + "latency_ms": 15221.457, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-f2b275130ace", + "path_leak": true, + "required_files": [ + "fleet-ops/config/marketing-program.json" + ], + "found": [], + "missed": [ + "fleet-ops/config/marketing-program.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1678, + "latency_ms": 2404.945, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-f7557d39e6e6", + "path_leak": false, + "required_files": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "found": [], + "missed": [ + "foundry/apps/public/public-directory/public/_redirects", + "foundry/ops/test/public-products.test.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1678, + "latency_ms": 13940.945, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-f8b07be6f191", + "path_leak": true, + "required_files": [ + "apps/mobile/app.json", + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "found": [ + "apps/mobile/app.json" + ], + "missed": [ + "apps/mobile/plugins/with-ios-pod-deployment-target.js", + "apps/mobile/test/ios-pod-deployment-target-plugin.test.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 4627, + "latency_ms": 869.188, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "site-health-fa0e527907ad", + "path_leak": true, + "required_files": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "found": [ + "foundry/ops/skills/agent-ready/scripts/agent-index-audit.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1672, + "latency_ms": 21364.845, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/V-today-little-log.json b/benchmarks/context-retrieval/results/V-today-little-log.json new file mode 100644 index 00000000..34fba8b3 --- /dev/null +++ b/benchmarks/context-retrieval/results/V-today-little-log.json @@ -0,0 +1,4942 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "today-little-log", + "head": "93b48d03329b74df043f0b45468f96ac21197c41" + }, + "tier": "small", + "tier_code_files": 117, + "corpus_counts": { + "cases": 41, + "path_leak": 21, + "baseline_blind": 20, + "multi_file": 15 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.4338, + "control": 0.0035, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "CSS import order and PostCSS conflict", + "revision": "0458fb2e26f9d78c20c824b75b62f3c8cc5a9592", + "returned": [ + "src/pwa.ts", + "src/components/ui/hover-card.tsx", + "src/components/ui/toast.tsx", + "src/components/ui/sheet.tsx", + "src/components/ui/button.tsx" + ] + }, + { + "provider_id": "random-files", + "query": "pre-push runs tsc + build, not just lint", + "revision": "9631951e22885658c3fa72897305aff4bd3690a8", + "returned": [ + "src/components/ui/label.tsx", + "src/components/StreakBadge.tsx", + "renovate.json", + "src/lib/auth-client.ts", + "drizzle/migrations/0002_urgency_wave_b.sql" + ] + }, + { + "provider_id": "random-files", + "query": "consolidate sidebar, tolerant /api/profiles, craft-hours widget", + "revision": "4a09700d0f7f1b5afad3dd6136090f448403bddb", + "returned": [ + "src/vite-env.d.ts", + "src/pages/Goals.tsx", + "src/components/LifeScoreBadge.tsx", + "api/_lib/auth-instance.ts", + "src/pages/Index.tsx" + ] + }, + { + "provider_id": "churn-ranked", + "query": "vendor @saas-maker/ai so Vercel builds", + "revision": "b297f57a34899bbca5463d4df60bc7acf40ba3c4", + "returned": [ + "src/pages/Index.tsx", + "package.json", + "src/App.tsx", + "src/components/AppSidebar.tsx", + "src/components/ScheduleMaker.tsx" + ] + }, + { + "provider_id": "churn-ranked", + "query": "route /api/auth/* subpaths to api/auth.ts", + "revision": "d3f3c964ec945f8b01a6c55acfe2fd19755a837a", + "returned": [ + "src/pages/Index.tsx", + "package.json", + "src/App.tsx", + "src/components/AppSidebar.tsx", + "vercel.json" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "CSS import order and PostCSS conflict", + "revision": "0458fb2e26f9d78c20c824b75b62f3c8cc5a9592", + "returned": [ + "postcss.config.js", + "src/components/saasmaker-feedback.tsx", + "src/main.tsx", + "src/index.css", + "api/_lib/schema.ts" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "add api/tsconfig.json with module:commonjs for Vercel", + "revision": "d43e7471f1ba1956cb4cfc1882cdd1feb4f67feb", + "returned": [ + "tsconfig.app.json", + "tsconfig.json", + "tsconfig.node.json", + "docs/plans/2026-02-25-chatbot-design.md", + "vercel.json" + ] + }, + { + "provider_id": "graphify", + "query": "use single auth.ts with rewrite for Better Auth catch-all", + "revision": "62d9d24a111619497a4398d2cba81a00e674bfe6", + "returned": [ + "api/_lib/auth.ts", + "src/pages/Auth.tsx", + "docs/plans/2026-02-27-better-auth-migration-plan.md", + "src/components/ErrorBoundary.tsx", + "vercel.json" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 41, + "mean_recall_at_5": 0.0035, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0157, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.097, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0035, + "mean_recall_at_16000_tokens": 0.0401, + "mean_precision_at_10": 0.0073, + "zero_hit_rate": 0.8293, + "median_tokens_delivered": 17981, + "median_latency_ms": 12.147, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 41, + "mean_recall_at_5": 0.0035, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0157, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.097, + "full_recall_at_20": 0.0732, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0035, + "mean_recall_at_16000_tokens": 0.0401, + "mean_precision_at_10": 0.0073, + "zero_hit_rate": 0.8293, + "median_tokens_delivered": 17981, + "median_latency_ms": 12.147, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.0068, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0187, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1298, + "full_recall_at_20": 0.0952, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0068, + "mean_recall_at_16000_tokens": 0.0663, + "mean_precision_at_10": 0.0095, + "zero_hit_rate": 0.7619, + "median_tokens_delivered": 16346, + "median_latency_ms": 12.271, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 20, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0125, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0625, + "full_recall_at_20": 0.05, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0125, + "mean_precision_at_10": 0.005, + "zero_hit_rate": 0.9, + "median_tokens_delivered": 19008.5, + "median_latency_ms": 11.966, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 41 + }, + "resources": { + "baseline_rss_mb": 27074.8, + "peak_rss_mb": 27183, + "delta_rss_mb": 108.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 41, + "mean_recall_at_5": 0.0801, + "full_recall_at_5": 0.0244, + "mean_recall_at_10": 0.2041, + "full_recall_at_10": 0.122, + "mean_recall_at_20": 0.3017, + "full_recall_at_20": 0.2195, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0157, + "mean_precision_at_10": 0.0341, + "zero_hit_rate": 0.561, + "median_tokens_delivered": 140450, + "median_latency_ms": 18.363, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 41, + "mean_recall_at_5": 0.0801, + "full_recall_at_5": 0.0244, + "mean_recall_at_10": 0.2041, + "full_recall_at_10": 0.122, + "mean_recall_at_20": 0.3017, + "full_recall_at_20": 0.2195, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0157, + "mean_precision_at_10": 0.0341, + "zero_hit_rate": 0.561, + "median_tokens_delivered": 140450, + "median_latency_ms": 18.363, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.0612, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.208, + "full_recall_at_10": 0.0952, + "mean_recall_at_20": 0.339, + "full_recall_at_20": 0.2381, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0068, + "mean_precision_at_10": 0.0429, + "zero_hit_rate": 0.4762, + "median_tokens_delivered": 217508, + "median_latency_ms": 18.145, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 20, + "mean_recall_at_5": 0.1, + "full_recall_at_5": 0.05, + "mean_recall_at_10": 0.2, + "full_recall_at_10": 0.15, + "mean_recall_at_20": 0.2625, + "full_recall_at_20": 0.2, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.025, + "mean_precision_at_10": 0.025, + "zero_hit_rate": 0.65, + "median_tokens_delivered": 139356.5, + "median_latency_ms": 18.6485, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 41 + }, + "resources": { + "baseline_rss_mb": 27152.8, + "peak_rss_mb": 27282.2, + "delta_rss_mb": 129.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 41, + "mean_recall_at_5": 0.3307, + "full_recall_at_5": 0.2683, + "mean_recall_at_10": 0.3545, + "full_recall_at_10": 0.2927, + "mean_recall_at_20": 0.4338, + "full_recall_at_20": 0.3659, + "mean_recall_at_1000_tokens": 0.4338, + "mean_recall_at_4000_tokens": 0.4338, + "mean_recall_at_16000_tokens": 0.4338, + "mean_precision_at_10": 0.0829, + "zero_hit_rate": 0.4878, + "median_tokens_delivered": 619, + "median_latency_ms": 193.273, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 41, + "mean_recall_at_5": 0.3307, + "full_recall_at_5": 0.2683, + "mean_recall_at_10": 0.3545, + "full_recall_at_10": 0.2927, + "mean_recall_at_20": 0.4338, + "full_recall_at_20": 0.3659, + "mean_recall_at_1000_tokens": 0.4338, + "mean_recall_at_4000_tokens": 0.4338, + "mean_recall_at_16000_tokens": 0.4338, + "mean_precision_at_10": 0.0829, + "zero_hit_rate": 0.4878, + "median_tokens_delivered": 619, + "median_latency_ms": 193.273, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.479, + "full_recall_at_5": 0.381, + "mean_recall_at_10": 0.5017, + "full_recall_at_10": 0.4286, + "mean_recall_at_20": 0.5969, + "full_recall_at_20": 0.5238, + "mean_recall_at_1000_tokens": 0.5969, + "mean_recall_at_4000_tokens": 0.5969, + "mean_recall_at_16000_tokens": 0.5969, + "mean_precision_at_10": 0.0857, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 726, + "median_latency_ms": 197.226, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 20, + "mean_recall_at_5": 0.175, + "full_recall_at_5": 0.15, + "mean_recall_at_10": 0.2, + "full_recall_at_10": 0.15, + "mean_recall_at_20": 0.2625, + "full_recall_at_20": 0.2, + "mean_recall_at_1000_tokens": 0.2625, + "mean_recall_at_4000_tokens": 0.2625, + "mean_recall_at_16000_tokens": 0.2625, + "mean_precision_at_10": 0.08, + "zero_hit_rate": 0.7, + "median_tokens_delivered": 571.5, + "median_latency_ms": 192.4615, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 41 + }, + "resources": { + "baseline_rss_mb": 27264.2, + "peak_rss_mb": 28772.7, + "delta_rss_mb": 1508.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 175759360 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 41, + "mean_recall_at_5": 0.3496, + "full_recall_at_5": 0.3171, + "mean_recall_at_10": 0.4141, + "full_recall_at_10": 0.3659, + "mean_recall_at_20": 0.4262, + "full_recall_at_20": 0.3902, + "mean_recall_at_1000_tokens": 0.2677, + "mean_recall_at_4000_tokens": 0.4262, + "mean_recall_at_16000_tokens": 0.4262, + "mean_precision_at_10": 0.0714, + "zero_hit_rate": 0.5122, + "median_tokens_delivered": 1678, + "median_latency_ms": 1710.031, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 41, + "mean_recall_at_5": 0.3496, + "full_recall_at_5": 0.3171, + "mean_recall_at_10": 0.4141, + "full_recall_at_10": 0.3659, + "mean_recall_at_20": 0.4262, + "full_recall_at_20": 0.3902, + "mean_recall_at_1000_tokens": 0.2677, + "mean_recall_at_4000_tokens": 0.4262, + "mean_recall_at_16000_tokens": 0.4262, + "mean_precision_at_10": 0.0714, + "zero_hit_rate": 0.5122, + "median_tokens_delivered": 1678, + "median_latency_ms": 1710.031, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 21, + "mean_recall_at_5": 0.4921, + "full_recall_at_5": 0.4286, + "mean_recall_at_10": 0.5703, + "full_recall_at_10": 0.4762, + "mean_recall_at_20": 0.5941, + "full_recall_at_20": 0.5238, + "mean_recall_at_1000_tokens": 0.3798, + "mean_recall_at_4000_tokens": 0.5941, + "mean_recall_at_16000_tokens": 0.5941, + "mean_precision_at_10": 0.0965, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 1681, + "median_latency_ms": 1744.764, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 20, + "mean_recall_at_5": 0.2, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.25, + "full_recall_at_10": 0.25, + "mean_recall_at_20": 0.25, + "full_recall_at_20": 0.25, + "mean_recall_at_1000_tokens": 0.15, + "mean_recall_at_4000_tokens": 0.25, + "mean_recall_at_16000_tokens": 0.25, + "mean_precision_at_10": 0.045, + "zero_hit_rate": 0.75, + "median_tokens_delivered": 1677, + "median_latency_ms": 1615.9895, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 41 + }, + "resources": { + "baseline_rss_mb": 27265.6, + "peak_rss_mb": 28777, + "delta_rss_mb": 1511.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 32817152 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "today-little-log-026ffeba899f", + "path_leak": true, + "required_files": [ + "api/auth.ts" + ], + "found": [ + "api/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 18523, + "latency_ms": 26.192, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-0458fb2e26f9", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105308, + "latency_ms": 12.242, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-085a8f98f4e3", + "path_leak": false, + "required_files": [ + "vite.config.ts" + ], + "found": [ + "vite.config.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 109597, + "latency_ms": 14.538, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-0d69b26aa12b", + "path_leak": true, + "required_files": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "found": [], + "missed": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13251, + "latency_ms": 15.282, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-1a06cf12d177", + "path_leak": false, + "required_files": [ + "functions/api/[resource].ts" + ], + "found": [], + "missed": [ + "functions/api/[resource].ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 102343, + "latency_ms": 13.273, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-21e91a0cf7d1", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts" + ], + "found": [], + "missed": [ + "api/_lib/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19013, + "latency_ms": 12.875, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-2576f31c791b", + "path_leak": true, + "required_files": [ + "eslint.config.js" + ], + "found": [], + "missed": [ + "eslint.config.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20860, + "latency_ms": 12.53, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-311f04610c97", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "api/auth.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19074, + "latency_ms": 12.019, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-3648fa9a9b79", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10438, + "latency_ms": 11.24, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-3822e19e5360", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13637, + "latency_ms": 12.465, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-432012502419", + "path_leak": true, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12447, + "latency_ms": 11.651, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-44e885050a3a", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15916, + "latency_ms": 12.305, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-49d25eea3efb", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13348, + "latency_ms": 12.485, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-4a06c9457671", + "path_leak": true, + "required_files": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "found": [], + "missed": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27482, + "latency_ms": 11.225, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-4c2bf18061a1", + "path_leak": false, + "required_files": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "found": [ + "postcss.config.js" + ], + "missed": [ + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 6, + "tokens_delivered": 14015, + "latency_ms": 11.518, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-5ff55ed07b2f", + "path_leak": true, + "required_files": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17981, + "latency_ms": 11.062, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-6d6487fa6ee3", + "path_leak": false, + "required_files": [ + "src/components/AppLayout.tsx" + ], + "found": [], + "missed": [ + "src/components/AppLayout.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123591, + "latency_ms": 11.445, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-6e8bd0230c30", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10493, + "latency_ms": 11.923, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-799f599a32dd", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 104945, + "latency_ms": 12.182, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-7ad692618d92", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123169, + "latency_ms": 12.147, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-83fa639f21f8", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "found": [], + "missed": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 111966, + "latency_ms": 10.917, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-8a71258e7d16", + "path_leak": true, + "required_files": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16346, + "latency_ms": 12.611, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-8b29bbd28725", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19262, + "latency_ms": 13.576, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-93b48d03329b", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10087, + "latency_ms": 13.426, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-9b9d47f48e0b", + "path_leak": false, + "required_files": [ + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12873, + "latency_ms": 15.064, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-9e21586af581", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "found": [ + "src/components/Navbar.tsx" + ], + "missed": [ + ".husky/pre-push", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 10, + "tokens_delivered": 13203, + "latency_ms": 13.297, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-af8c1b0d433b", + "path_leak": false, + "required_files": [ + "api/_schema.ts", + "api/debug.ts" + ], + "found": [], + "missed": [ + "api/_schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 9739, + "latency_ms": 10.778, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-b26636817b1b", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 16299, + "latency_ms": 10.767, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-b297f57a3489", + "path_leak": true, + "required_files": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "found": [ + "src/pages/Index.tsx" + ], + "missed": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Schedule.tsx" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.1429, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1429, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 5, + "tokens_delivered": 13293, + "latency_ms": 11.297, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-bc2139cca1ca", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12829, + "latency_ms": 11.878, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-bd456960c205", + "path_leak": false, + "required_files": [ + "src/components/EntryEditor.tsx" + ], + "found": [], + "missed": [ + "src/components/EntryEditor.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13606, + "latency_ms": 11.849, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-be8f1bcad0c1", + "path_leak": false, + "required_files": [ + ".gitignore" + ], + "found": [], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23062, + "latency_ms": 12.507, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-c1da954218b5", + "path_leak": false, + "required_files": [ + "vitest.config.ts" + ], + "found": [], + "missed": [ + "vitest.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 100717, + "latency_ms": 11.381, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-c1dbe746df7c", + "path_leak": false, + "required_files": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14777, + "latency_ms": 11.732, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-cda10389aa0f", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10942, + "latency_ms": 10.842, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-d3f3c964ec94", + "path_leak": true, + "required_files": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 147928, + "latency_ms": 12.737, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-d543600425f6", + "path_leak": false, + "required_files": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "found": [], + "missed": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13672, + "latency_ms": 11.418, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-dcca049fcd7b", + "path_leak": false, + "required_files": [ + "src/index.css" + ], + "found": [], + "missed": [ + "src/index.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 108092, + "latency_ms": 12.009, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-e2fd6e5d8453", + "path_leak": false, + "required_files": [ + ".saasmaker.json" + ], + "found": [], + "missed": [ + ".saasmaker.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 116794, + "latency_ms": 10.766, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-e37951b10026", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "api/auth.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20848, + "latency_ms": 12.271, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "today-little-log-f81cc54457ab", + "path_leak": false, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18755, + "latency_ms": 12.511, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-026ffeba899f", + "path_leak": true, + "required_files": [ + "api/auth.ts" + ], + "found": [], + "missed": [ + "api/auth.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 217508, + "latency_ms": 21.847, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-0458fb2e26f9", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 250904, + "latency_ms": 18.805, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-085a8f98f4e3", + "path_leak": false, + "required_files": [ + "vite.config.ts" + ], + "found": [], + "missed": [ + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 218109, + "latency_ms": 19.826, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-0d69b26aa12b", + "path_leak": true, + "required_files": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "vendor/saas-maker-ai/package.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 253670, + "latency_ms": 19.791, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-1a06cf12d177", + "path_leak": false, + "required_files": [ + "functions/api/[resource].ts" + ], + "found": [], + "missed": [ + "functions/api/[resource].ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 113820, + "latency_ms": 24.697, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-21e91a0cf7d1", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts" + ], + "found": [ + "api/_lib/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 129831, + "latency_ms": 17.728, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-2576f31c791b", + "path_leak": true, + "required_files": [ + "eslint.config.js" + ], + "found": [], + "missed": [ + "eslint.config.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 121393, + "latency_ms": 18.368, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-311f04610c97", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "api/auth.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 217508, + "latency_ms": 19.656, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-3648fa9a9b79", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 217726, + "latency_ms": 18.835, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-3822e19e5360", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 130907, + "latency_ms": 18.13, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-432012502419", + "path_leak": true, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 217726, + "latency_ms": 18.34, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-44e885050a3a", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 127762, + "latency_ms": 18.363, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-49d25eea3efb", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 130201, + "latency_ms": 17.48, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-4a06c9457671", + "path_leak": true, + "required_files": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "found": [], + "missed": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 131648, + "latency_ms": 18.836, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-4c2bf18061a1", + "path_leak": false, + "required_files": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "found": [ + "vite.config.ts" + ], + "missed": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 161970, + "latency_ms": 19.287, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-5ff55ed07b2f", + "path_leak": true, + "required_files": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 121393, + "latency_ms": 21.604, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-6d6487fa6ee3", + "path_leak": false, + "required_files": [ + "src/components/AppLayout.tsx" + ], + "found": [ + "src/components/AppLayout.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 138033, + "latency_ms": 18.772, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-6e8bd0230c30", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 134327, + "latency_ms": 20.609, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-799f599a32dd", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 130907, + "latency_ms": 19.336, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-7ad692618d92", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 217726, + "latency_ms": 18.764, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-83fa639f21f8", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "found": [ + "api/_lib/auth.ts" + ], + "missed": [ + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 130172, + "latency_ms": 17.771, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-8a71258e7d16", + "path_leak": true, + "required_files": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 219296, + "latency_ms": 19.417, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-8b29bbd28725", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 141649, + "latency_ms": 19.252, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-93b48d03329b", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140248, + "latency_ms": 23.387, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-9b9d47f48e0b", + "path_leak": false, + "required_files": [ + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 134115, + "latency_ms": 15.736, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-9e21586af581", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "found": [ + "src/components/Navbar.tsx" + ], + "missed": [ + ".husky/pre-push", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 114964, + "latency_ms": 17.13, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-af8c1b0d433b", + "path_leak": false, + "required_files": [ + "api/_schema.ts", + "api/debug.ts" + ], + "found": [ + "api/debug.ts" + ], + "missed": [ + "api/_schema.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 129252, + "latency_ms": 16.082, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-b26636817b1b", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 130907, + "latency_ms": 17.322, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-b297f57a3489", + "path_leak": true, + "required_files": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "found": [ + "src/components/AppSidebar.tsx", + "src/pages/Index.tsx" + ], + "missed": [ + "api/profiles.ts", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Schedule.tsx" + ], + "recall_at_5": 0.2857, + "precision_at_5": 0.4, + "recall_at_10": 0.2857, + "precision_at_10": 0.2, + "recall_at_20": 0.2857, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 1, + "tokens_delivered": 255668, + "latency_ms": 18.145, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-bc2139cca1ca", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [ + "tests/prod-auth.spec.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 254337, + "latency_ms": 18.14, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-bd456960c205", + "path_leak": false, + "required_files": [ + "src/components/EntryEditor.tsx" + ], + "found": [], + "missed": [ + "src/components/EntryEditor.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 218528, + "latency_ms": 16.277, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-be8f1bcad0c1", + "path_leak": false, + "required_files": [ + ".gitignore" + ], + "found": [], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 126863, + "latency_ms": 17.951, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-c1da954218b5", + "path_leak": false, + "required_files": [ + "vitest.config.ts" + ], + "found": [], + "missed": [ + "vitest.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140450, + "latency_ms": 22.234, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-c1dbe746df7c", + "path_leak": false, + "required_files": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 129189, + "latency_ms": 18.533, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-cda10389aa0f", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 217718, + "latency_ms": 17.853, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-d3f3c964ec94", + "path_leak": true, + "required_files": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 254316, + "latency_ms": 17.39, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-d543600425f6", + "path_leak": false, + "required_files": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "found": [ + "src/pages/Index.tsx" + ], + "missed": [ + "src/main.tsx" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 138263, + "latency_ms": 17.939, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-dcca049fcd7b", + "path_leak": false, + "required_files": [ + "src/index.css" + ], + "found": [], + "missed": [ + "src/index.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 250904, + "latency_ms": 17.293, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-e2fd6e5d8453", + "path_leak": false, + "required_files": [ + ".saasmaker.json" + ], + "found": [], + "missed": [ + ".saasmaker.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 219008, + "latency_ms": 16.978, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-e37951b10026", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [ + "api/auth.ts", + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 217966, + "latency_ms": 16.744, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "today-little-log-f81cc54457ab", + "path_leak": false, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 253752, + "latency_ms": 16.568, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-026ffeba899f", + "path_leak": true, + "required_files": [ + "api/auth.ts" + ], + "found": [ + "api/auth.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 794, + "latency_ms": 459.083, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-0458fb2e26f9", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 619, + "latency_ms": 188.275, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-085a8f98f4e3", + "path_leak": false, + "required_files": [ + "vite.config.ts" + ], + "found": [], + "missed": [ + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 300, + "latency_ms": 158.537, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-0d69b26aa12b", + "path_leak": true, + "required_files": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "found": [], + "missed": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 726, + "latency_ms": 229.365, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-1a06cf12d177", + "path_leak": false, + "required_files": [ + "functions/api/[resource].ts" + ], + "found": [], + "missed": [ + "functions/api/[resource].ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 558, + "latency_ms": 191.65, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-21e91a0cf7d1", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts" + ], + "found": [ + "api/_lib/auth.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 576, + "latency_ms": 223.067, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-2576f31c791b", + "path_leak": true, + "required_files": [ + "eslint.config.js" + ], + "found": [ + "eslint.config.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 496, + "latency_ms": 227.115, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-311f04610c97", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "api/auth.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 750, + "latency_ms": 167.425, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-3648fa9a9b79", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [ + "scripts/bundle-api.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 51, + "latency_ms": 164.3, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-3822e19e5360", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 766, + "latency_ms": 201.995, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-432012502419", + "path_leak": true, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [ + "scripts/bundle-api.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 445, + "latency_ms": 160.589, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-44e885050a3a", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 856, + "latency_ms": 193.273, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-49d25eea3efb", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 18, + "tokens_delivered": 756, + "latency_ms": 166.294, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-4a06c9457671", + "path_leak": true, + "required_files": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "found": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 756, + "latency_ms": 166.387, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-4c2bf18061a1", + "path_leak": false, + "required_files": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "found": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts" + ], + "missed": [ + "vite.config.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 722, + "latency_ms": 163.176, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-5ff55ed07b2f", + "path_leak": true, + "required_files": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 813, + "latency_ms": 199.688, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-6d6487fa6ee3", + "path_leak": false, + "required_files": [ + "src/components/AppLayout.tsx" + ], + "found": [ + "src/components/AppLayout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 372, + "latency_ms": 146.981, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-6e8bd0230c30", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 603, + "latency_ms": 206.647, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-799f599a32dd", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 724, + "latency_ms": 177.748, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-7ad692618d92", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 764, + "latency_ms": 189.578, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-83fa639f21f8", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "found": [ + "api/_lib/auth.ts", + "api/_lib/db.ts" + ], + "missed": [ + "api/_lib/schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 536, + "latency_ms": 153.842, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-8a71258e7d16", + "path_leak": true, + "required_files": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "found": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 822, + "latency_ms": 171.66, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-8b29bbd28725", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 190.163, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-93b48d03329b", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts" + ], + "found": [ + "tests/prod-auth.spec.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 733, + "latency_ms": 197.226, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-9b9d47f48e0b", + "path_leak": false, + "required_files": [ + "src/hooks/useChat.ts" + ], + "found": [], + "missed": [ + "src/hooks/useChat.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 667, + "latency_ms": 149.413, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-9e21586af581", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "found": [ + ".husky/pre-push" + ], + "missed": [ + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 573, + "latency_ms": 160.035, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-af8c1b0d433b", + "path_leak": false, + "required_files": [ + "api/_schema.ts", + "api/debug.ts" + ], + "found": [ + "api/debug.ts" + ], + "missed": [ + "api/_schema.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 595, + "latency_ms": 170.039, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-b26636817b1b", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 680, + "latency_ms": 175.83, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-b297f57a3489", + "path_leak": true, + "required_files": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "found": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx" + ], + "missed": [ + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "recall_at_5": 0.1429, + "precision_at_5": 0.2, + "recall_at_10": 0.2857, + "precision_at_10": 0.2, + "recall_at_20": 0.2857, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.2857, + "recall_at_4000_tokens": 0.2857, + "recall_at_16000_tokens": 0.2857, + "first_hit_rank": 2, + "tokens_delivered": 726, + "latency_ms": 227.859, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-bc2139cca1ca", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 502, + "latency_ms": 328.754, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-bd456960c205", + "path_leak": false, + "required_files": [ + "src/components/EntryEditor.tsx" + ], + "found": [], + "missed": [ + "src/components/EntryEditor.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 585, + "latency_ms": 281.516, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-be8f1bcad0c1", + "path_leak": false, + "required_files": [ + ".gitignore" + ], + "found": [], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 482, + "latency_ms": 325.702, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-c1da954218b5", + "path_leak": false, + "required_files": [ + "vitest.config.ts" + ], + "found": [], + "missed": [ + "vitest.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 203, + "latency_ms": 327.003, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-c1dbe746df7c", + "path_leak": false, + "required_files": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 329, + "latency_ms": 383.223, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-cda10389aa0f", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 19, + "tokens_delivered": 832, + "latency_ms": 309.325, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-d3f3c964ec94", + "path_leak": true, + "required_files": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 283, + "latency_ms": 266.226, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-d543600425f6", + "path_leak": false, + "required_files": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "found": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 670, + "latency_ms": 233.922, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-dcca049fcd7b", + "path_leak": false, + "required_files": [ + "src/index.css" + ], + "found": [ + "src/index.css" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 147, + "latency_ms": 226.526, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-e2fd6e5d8453", + "path_leak": false, + "required_files": [ + ".saasmaker.json" + ], + "found": [], + "missed": [ + ".saasmaker.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 126, + "latency_ms": 283.836, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-e37951b10026", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [ + "api/auth.ts" + ], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 685, + "latency_ms": 178.523, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "today-little-log-f81cc54457ab", + "path_leak": false, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 780, + "latency_ms": 244.901, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-026ffeba899f", + "path_leak": true, + "required_files": [ + "api/auth.ts" + ], + "found": [ + "api/auth.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 1572, + "latency_ms": 1738.706, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-0458fb2e26f9", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4058, + "latency_ms": 1688.082, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-085a8f98f4e3", + "path_leak": false, + "required_files": [ + "vite.config.ts" + ], + "found": [], + "missed": [ + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1676, + "latency_ms": 1881.701, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-0d69b26aa12b", + "path_leak": true, + "required_files": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "found": [], + "missed": [ + "package.json", + "vendor/saas-maker-ai/package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1675, + "latency_ms": 2127.56, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-1a06cf12d177", + "path_leak": false, + "required_files": [ + "functions/api/[resource].ts" + ], + "found": [], + "missed": [ + "functions/api/[resource].ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1670, + "latency_ms": 1604.354, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-21e91a0cf7d1", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts" + ], + "found": [ + "api/_lib/auth.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1682, + "latency_ms": 1671.712, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-2576f31c791b", + "path_leak": true, + "required_files": [ + "eslint.config.js" + ], + "found": [ + "eslint.config.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1675, + "latency_ms": 2611.028, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-311f04610c97", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [ + "api/auth.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1684, + "latency_ms": 1505.604, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-3648fa9a9b79", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 377, + "latency_ms": 1889.204, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-3822e19e5360", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 138, + "latency_ms": 1744.764, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-432012502419", + "path_leak": true, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [ + "scripts/bundle-api.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1306, + "latency_ms": 1740.205, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-44e885050a3a", + "path_leak": false, + "required_files": [ + ".husky/pre-push" + ], + "found": [], + "missed": [ + ".husky/pre-push" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 162, + "latency_ms": 1734.862, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-49d25eea3efb", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3164, + "latency_ms": 1690.343, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-4a06c9457671", + "path_leak": true, + "required_files": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts", + "tests/smoke.spec.ts" + ], + "found": [ + "tests/smoke.spec.ts" + ], + "missed": [ + "tests/interactive.spec.ts", + "tests/prod-auth.spec.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 1677, + "latency_ms": 1697.836, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-4c2bf18061a1", + "path_leak": false, + "required_files": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "found": [], + "missed": [ + "postcss.config.js", + "src/index.css", + "tailwind.config.ts", + "vite.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 142, + "latency_ms": 1561.117, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-5ff55ed07b2f", + "path_leak": true, + "required_files": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "found": [], + "missed": [ + "functions/api/auth/[[all]].ts", + "wrangler.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1710, + "latency_ms": 2022.617, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-6d6487fa6ee3", + "path_leak": false, + "required_files": [ + "src/components/AppLayout.tsx" + ], + "found": [ + "src/components/AppLayout.tsx" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3100, + "latency_ms": 1454.52, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-6e8bd0230c30", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2488, + "latency_ms": 1664.492, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-799f599a32dd", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 840, + "latency_ms": 3576.011, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-7ad692618d92", + "path_leak": false, + "required_files": [ + "scripts/bundle-api.mjs" + ], + "found": [ + "scripts/bundle-api.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 6731, + "latency_ms": 1723.42, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-83fa639f21f8", + "path_leak": true, + "required_files": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "found": [], + "missed": [ + "api/_lib/auth.ts", + "api/_lib/db.ts", + "api/_lib/schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3508, + "latency_ms": 1860.691, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-8a71258e7d16", + "path_leak": true, + "required_files": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "found": [ + "src/components/ChatBot.tsx", + "src/hooks/useChat.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1700, + "latency_ms": 1808.503, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-8b29bbd28725", + "path_leak": false, + "required_files": [ + "index.html" + ], + "found": [], + "missed": [ + "index.html" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 458, + "latency_ms": 2046.089, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-93b48d03329b", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts" + ], + "found": [ + "tests/prod-auth.spec.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5188, + "latency_ms": 2057.649, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-9b9d47f48e0b", + "path_leak": false, + "required_files": [ + "src/hooks/useChat.ts" + ], + "found": [ + "src/hooks/useChat.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 1667, + "latency_ms": 1487.316, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-9e21586af581", + "path_leak": true, + "required_files": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "found": [], + "missed": [ + ".husky/pre-push", + "src/components/Navbar.tsx", + "src/components/ui/calendar.tsx", + "src/db/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1466, + "latency_ms": 1516.076, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-af8c1b0d433b", + "path_leak": false, + "required_files": [ + "api/_schema.ts", + "api/debug.ts" + ], + "found": [], + "missed": [ + "api/_schema.ts", + "api/debug.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1679, + "latency_ms": 1581.477, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-b26636817b1b", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 7081, + "latency_ms": 1570.471, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-b297f57a3489", + "path_leak": true, + "required_files": [ + "api/profiles.ts", + "src/components/AppSidebar.tsx", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "found": [ + "src/components/AppSidebar.tsx" + ], + "missed": [ + "api/profiles.ts", + "src/components/CraftHoursWidget.tsx", + "src/components/NowRecommender.tsx", + "src/components/QuickLogFab.tsx", + "src/pages/Index.tsx", + "src/pages/Schedule.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1429, + "precision_at_10": 0.1, + "recall_at_20": 0.1429, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.1429, + "recall_at_4000_tokens": 0.1429, + "recall_at_16000_tokens": 0.1429, + "first_hit_rank": 10, + "tokens_delivered": 1671, + "latency_ms": 1849.558, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-bc2139cca1ca", + "path_leak": true, + "required_files": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "tests/prod-auth.spec.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1674, + "latency_ms": 1802.437, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-bd456960c205", + "path_leak": false, + "required_files": [ + "src/components/EntryEditor.tsx" + ], + "found": [], + "missed": [ + "src/components/EntryEditor.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 9033, + "latency_ms": 1305.353, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-be8f1bcad0c1", + "path_leak": false, + "required_files": [ + ".gitignore" + ], + "found": [], + "missed": [ + ".gitignore" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1678, + "latency_ms": 1710.031, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-c1da954218b5", + "path_leak": false, + "required_files": [ + "vitest.config.ts" + ], + "found": [], + "missed": [ + "vitest.config.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1710, + "latency_ms": 1462.72, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-c1dbe746df7c", + "path_leak": false, + "required_files": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "found": [], + "missed": [ + ".gitignore", + "scripts/bundle-api.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8212, + "latency_ms": 1138.185, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-cda10389aa0f", + "path_leak": true, + "required_files": [ + "vercel.json" + ], + "found": [ + "vercel.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 7746, + "latency_ms": 2210.542, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-d3f3c964ec94", + "path_leak": true, + "required_files": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "found": [], + "missed": [ + "tests/auth-button.spec.ts", + "tests/interactive.spec.ts", + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1681, + "latency_ms": 1909.242, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-d543600425f6", + "path_leak": false, + "required_files": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "found": [], + "missed": [ + "src/main.tsx", + "src/pages/Index.tsx" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8261, + "latency_ms": 1627.625, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-dcca049fcd7b", + "path_leak": false, + "required_files": [ + "src/index.css" + ], + "found": [], + "missed": [ + "src/index.css" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 229, + "latency_ms": 1332.43, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-e2fd6e5d8453", + "path_leak": false, + "required_files": [ + ".saasmaker.json" + ], + "found": [], + "missed": [ + ".saasmaker.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 756, + "latency_ms": 1486.072, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-e37951b10026", + "path_leak": true, + "required_files": [ + "api/auth.ts", + "vercel.json" + ], + "found": [ + "api/auth.ts" + ], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1674, + "latency_ms": 1245.369, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "today-little-log-f81cc54457ab", + "path_leak": false, + "required_files": [ + "vercel.json" + ], + "found": [], + "missed": [ + "vercel.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1685, + "latency_ms": 1922.642, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/V-what-it-takes-to-win.json b/benchmarks/context-retrieval/results/V-what-it-takes-to-win.json new file mode 100644 index 00000000..a9108293 --- /dev/null +++ b/benchmarks/context-retrieval/results/V-what-it-takes-to-win.json @@ -0,0 +1,2203 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "what-it-takes-to-win", + "head": "d3747e8cea4fd1d886ad5c54c09d6096d182352d" + }, + "tier": "small", + "tier_code_files": 65, + "corpus_counts": { + "cases": 14, + "path_leak": 9, + "baseline_blind": 5, + "multi_file": 9 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.5655, + "control": 0, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "Fix birth year estimation: add birth_year column to CSV, recover 195 filtered people", + "revision": "5460fddaeb2ce4baae19ad23051f2ccfa9291390", + "returned": [ + "src/pages/index.astro", + "src/pages/person/[id].astro", + ".github/workflows/ci.yml", + "src/data/archetypes.json", + "src/scripts/merge_research.py" + ] + }, + { + "provider_id": "random-files", + "query": "keep generated profile links audit-safe", + "revision": "25cca210c275ab0311e5c177f5399bab158b4114", + "returned": [ + "src/pages/methodology/index.astro", + "src/pages/insights/index.astro", + "src/pages/compare/index.astro", + "data/research/merge_report.json", + "src/data/archetypes.json" + ] + }, + { + "provider_id": "random-files", + "query": "Fix data quality: remove 6 duplicate IDs, cap 2393 leverage scores at 0-2", + "revision": "77787b7d957c37a1988bd747876d9fd686e5ac58", + "returned": [ + ".github/workflows/ci.yml", + "src/pages/person/[id].astro", + "src/data/archetypes.json", + "src/data/people.json", + "src/scripts/build_founder_engineer_queue.py" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix birth year estimation: add birth_year column to CSV, recover 195 filtered people", + "revision": "5460fddaeb2ce4baae19ad23051f2ccfa9291390", + "returned": [ + "src/data/people.json", + "data/research/merge_report.json", + "src/pages/compare/index.astro", + "src/data/archetypes.json", + "src/pages/person/[id].astro" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix person_id slugs for subagent-researched people", + "revision": "f784628dce33bc8e82d943fdfc2dbd45b26deed9", + "returned": [ + "src/data/people.json", + "data/research/merge_report.json", + ".github/workflows/ci.yml", + "astro.config.mjs", + "package-lock.json" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "satisfy research script quality checks", + "revision": "1ce2ba36ee066491a27801755037e5f8c68221a6", + "returned": [ + "src/scripts/check-code-health.mjs", + "public/scripts/posthog.js", + "src/scripts/merge_founder_research.mjs", + "src/scripts/merge_research.py", + "src/scripts/story-world.ts" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "Fix person_id slugs for subagent-researched people", + "revision": "f784628dce33bc8e82d943fdfc2dbd45b26deed9", + "returned": [ + "src/pages/person/[id].astro", + "src/scripts/charts.js", + "public/data/people.csv", + "src/data/people.csv", + "src/data/people.json" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "stop claiming a completed source audit on /coverage/", + "revision": "9bcd0d62e016122c615a87d356e5d47a930085ba", + "returned": [ + "src/scripts/audit_sources.mjs", + "src/data/methodology.md", + "quality/source-audit/audit-v1.json", + "quality/source-audit/REPORT.md", + "src/lib/coverage.ts" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 14, + "mean_recall_at_5": 0.1786, + "full_recall_at_5": 0.0714, + "mean_recall_at_10": 0.2738, + "full_recall_at_10": 0.0714, + "mean_recall_at_20": 0.4167, + "full_recall_at_20": 0.2143, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0357, + "mean_precision_at_10": 0.0643, + "zero_hit_rate": 0.4286, + "median_tokens_delivered": 468773, + "median_latency_ms": 13.1465, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 13, + "mean_recall_at_5": 0.1154, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2179, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3718, + "full_recall_at_20": 0.1538, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0385, + "mean_precision_at_10": 0.0615, + "zero_hit_rate": 0.4615, + "median_tokens_delivered": 280819, + "median_latency_ms": 13.407, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 1, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 6535110, + "median_latency_ms": 12.625, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.1667, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1667, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2778, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0556, + "mean_precision_at_10": 0.0556, + "zero_hit_rate": 0.5556, + "median_tokens_delivered": 255464, + "median_latency_ms": 14.229, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 5, + "mean_recall_at_5": 0.2, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.4667, + "full_recall_at_10": 0.2, + "mean_recall_at_20": 0.6667, + "full_recall_at_20": 0.6, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.08, + "zero_hit_rate": 0.2, + "median_tokens_delivered": 3297240, + "median_latency_ms": 12.625, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 14 + }, + "resources": { + "baseline_rss_mb": 27088, + "peak_rss_mb": 27112.5, + "delta_rss_mb": 24.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 14, + "mean_recall_at_5": 0.25, + "full_recall_at_5": 0.0714, + "mean_recall_at_10": 0.4583, + "full_recall_at_10": 0.2857, + "mean_recall_at_20": 0.756, + "full_recall_at_20": 0.5714, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0179, + "mean_precision_at_10": 0.1071, + "zero_hit_rate": 0.1429, + "median_tokens_delivered": 6735249, + "median_latency_ms": 22.991, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 13, + "mean_recall_at_5": 0.1923, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.4167, + "full_recall_at_10": 0.2308, + "mean_recall_at_20": 0.7372, + "full_recall_at_20": 0.5385, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0192, + "mean_precision_at_10": 0.1077, + "zero_hit_rate": 0.1538, + "median_tokens_delivered": 6709755, + "median_latency_ms": 22.891, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 1, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 6760802, + "median_latency_ms": 23.091, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.1667, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.3056, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.6204, + "full_recall_at_20": 0.3333, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0278, + "mean_precision_at_10": 0.0889, + "zero_hit_rate": 0.2222, + "median_tokens_delivered": 6709755, + "median_latency_ms": 25.604, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 5, + "mean_recall_at_5": 0.4, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.7333, + "full_recall_at_10": 0.6, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.14, + "zero_hit_rate": 0, + "median_tokens_delivered": 6760743, + "median_latency_ms": 21.5, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 14 + }, + "resources": { + "baseline_rss_mb": 27136.4, + "peak_rss_mb": 27161.3, + "delta_rss_mb": 25, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 14, + "mean_recall_at_5": 0.3184, + "full_recall_at_5": 0.2143, + "mean_recall_at_10": 0.378, + "full_recall_at_10": 0.2857, + "mean_recall_at_20": 0.4494, + "full_recall_at_20": 0.3571, + "mean_recall_at_1000_tokens": 0.4137, + "mean_recall_at_4000_tokens": 0.4494, + "mean_recall_at_16000_tokens": 0.4494, + "mean_precision_at_10": 0.0783, + "zero_hit_rate": 0.4286, + "median_tokens_delivered": 707.5, + "median_latency_ms": 596.7665, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 13, + "mean_recall_at_5": 0.3429, + "full_recall_at_5": 0.2308, + "mean_recall_at_10": 0.4071, + "full_recall_at_10": 0.3077, + "mean_recall_at_20": 0.484, + "full_recall_at_20": 0.3846, + "mean_recall_at_1000_tokens": 0.4455, + "mean_recall_at_4000_tokens": 0.484, + "mean_recall_at_16000_tokens": 0.484, + "mean_precision_at_10": 0.0844, + "zero_hit_rate": 0.3846, + "median_tokens_delivered": 706, + "median_latency_ms": 617.309, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 1, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 2895, + "median_latency_ms": 542.13, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.4954, + "full_recall_at_5": 0.3333, + "mean_recall_at_10": 0.588, + "full_recall_at_10": 0.4444, + "mean_recall_at_20": 0.6991, + "full_recall_at_20": 0.5556, + "mean_recall_at_1000_tokens": 0.6435, + "mean_recall_at_4000_tokens": 0.6991, + "mean_recall_at_16000_tokens": 0.6991, + "mean_precision_at_10": 0.1219, + "zero_hit_rate": 0.1111, + "median_tokens_delivered": 730, + "median_latency_ms": 732.119, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 5, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 298, + "median_latency_ms": 542.13, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 13, + "indexed-but-returned-nothing": 1 + }, + "resources": { + "baseline_rss_mb": 27148.4, + "peak_rss_mb": 28766, + "delta_rss_mb": 1617.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 91381760 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 14, + "mean_recall_at_5": 0.5179, + "full_recall_at_5": 0.4286, + "mean_recall_at_10": 0.5417, + "full_recall_at_10": 0.4286, + "mean_recall_at_20": 0.5655, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0.3631, + "mean_recall_at_4000_tokens": 0.5655, + "mean_recall_at_16000_tokens": 0.5655, + "mean_precision_at_10": 0.1401, + "zero_hit_rate": 0.3571, + "median_tokens_delivered": 2745.5, + "median_latency_ms": 2376.937, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 13, + "mean_recall_at_5": 0.4808, + "full_recall_at_5": 0.3846, + "mean_recall_at_10": 0.5064, + "full_recall_at_10": 0.3846, + "mean_recall_at_20": 0.5321, + "full_recall_at_20": 0.4615, + "mean_recall_at_1000_tokens": 0.391, + "mean_recall_at_4000_tokens": 0.5321, + "mean_recall_at_16000_tokens": 0.5321, + "mean_precision_at_10": 0.1432, + "zero_hit_rate": 0.3846, + "median_tokens_delivered": 2518, + "median_latency_ms": 2394.766, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 1, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0, + "median_tokens_delivered": 5197, + "median_latency_ms": 2355.442, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.5463, + "full_recall_at_5": 0.4444, + "mean_recall_at_10": 0.5463, + "full_recall_at_10": 0.4444, + "mean_recall_at_20": 0.5463, + "full_recall_at_20": 0.4444, + "mean_recall_at_1000_tokens": 0.3796, + "mean_recall_at_4000_tokens": 0.5463, + "mean_recall_at_16000_tokens": 0.5463, + "mean_precision_at_10": 0.1623, + "zero_hit_rate": 0.3333, + "median_tokens_delivered": 4162, + "median_latency_ms": 3160.813, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 5, + "mean_recall_at_5": 0.4667, + "full_recall_at_5": 0.4, + "mean_recall_at_10": 0.5333, + "full_recall_at_10": 0.4, + "mean_recall_at_20": 0.6, + "full_recall_at_20": 0.6, + "mean_recall_at_1000_tokens": 0.3333, + "mean_recall_at_4000_tokens": 0.6, + "mean_recall_at_16000_tokens": 0.6, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.4, + "median_tokens_delivered": 1697, + "median_latency_ms": 2355.442, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 13, + "indexed-but-returned-nothing": 1 + }, + "resources": { + "baseline_rss_mb": 28265.5, + "peak_rss_mb": 28740, + "delta_rss_mb": 474.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 6459392 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-3dd1b2f11b56", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 85064, + "latency_ms": 25.945, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-42de39f1e3e7", + "path_leak": true, + "required_files": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "found": [], + "missed": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 146049, + "latency_ms": 23.148, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-62d014f029c5", + "path_leak": true, + "required_files": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/build_dataset.mjs" + ], + "found": [ + "src/data/archetypes.json", + "src/scripts/build_dataset.mjs" + ], + "missed": [ + "src/data/people.csv", + "src/data/people.json" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 116883, + "latency_ms": 14.229, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-6e229e791c6c", + "path_leak": true, + "required_files": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "found": [ + "src/scripts/audit_discovery.mjs" + ], + "missed": [ + "src/scripts/generate_discovery_surfaces.mjs" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 70158, + "latency_ms": 13.407, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-77787b7d957c", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 3211331, + "latency_ms": 12.886, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-8105b178afd6", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "found": [ + "src/pages/index.astro" + ], + "missed": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 3297240, + "latency_ms": 12.203, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-9d6998dcea66", + "path_leak": true, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 280819, + "latency_ms": 12.308, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-a6e68b3385a5", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 3474579, + "latency_ms": 11.131, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-c699da18123d", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro" + ], + "found": [ + "src/layouts/Base.astro" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 6535110, + "latency_ms": 12.625, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-c9ac67e7bf08", + "path_leak": true, + "required_files": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "found": [ + "data/research/merge_report.json", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "missed": [ + "src/data/people.csv" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 255464, + "latency_ms": 11.941, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-d3747e8cea4f", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [], + "missed": [ + "src/scripts/marble-world.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1023991, + "latency_ms": 21.428, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-dbe87b68b695", + "path_leak": true, + "required_files": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "found": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.json" + ], + "missed": [ + "src/data/people.csv" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 6487453, + "latency_ms": 11.216, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-ec48ef20dcb2", + "path_leak": true, + "required_files": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "found": [], + "missed": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 193919, + "latency_ms": 15.239, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "what-it-takes-to-win-eca794d585ba", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [], + "missed": [ + "src/scripts/marble-world.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 656727, + "latency_ms": 15.327, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-3dd1b2f11b56", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 6765015, + "latency_ms": 23.427, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-42de39f1e3e7", + "path_leak": true, + "required_files": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "found": [], + "missed": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10025220, + "latency_ms": 31.217, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-62d014f029c5", + "path_leak": true, + "required_files": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/build_dataset.mjs" + ], + "found": [ + "src/data/archetypes.json", + "src/data/people.json", + "src/scripts/build_dataset.mjs" + ], + "missed": [ + "src/data/people.csv" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 1, + "tokens_delivered": 2687952, + "latency_ms": 22.891, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-6e229e791c6c", + "path_leak": true, + "required_files": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "found": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 6709755, + "latency_ms": 25.604, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-77787b7d957c", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 6427537, + "latency_ms": 19.692, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-8105b178afd6", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "found": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 6760743, + "latency_ms": 21.5, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-9d6998dcea66", + "path_leak": true, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 6709425, + "latency_ms": 22.774, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-a6e68b3385a5", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 3474209, + "latency_ms": 19.721, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-c699da18123d", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro" + ], + "found": [ + "src/layouts/Base.astro" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 6760802, + "latency_ms": 23.091, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-c9ac67e7bf08", + "path_leak": true, + "required_files": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "found": [ + "data/research/merge_report.json", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "missed": [ + "src/data/people.csv" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 255464, + "latency_ms": 18.718, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-d3747e8cea4f", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [ + "src/scripts/marble-world.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 10043630, + "latency_ms": 39.1, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-dbe87b68b695", + "path_leak": true, + "required_files": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "found": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.json" + ], + "missed": [ + "src/data/people.csv" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 1, + "tokens_delivered": 6427548, + "latency_ms": 20.542, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-ec48ef20dcb2", + "path_leak": true, + "required_files": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "found": [ + "src/scripts/audit_clarity.mjs" + ], + "missed": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 10025220, + "latency_ms": 27.117, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "what-it-takes-to-win-eca794d585ba", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [], + "missed": [ + "src/scripts/marble-world.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10037129, + "latency_ms": 31.244, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-3dd1b2f11b56", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 298, + "latency_ms": 1241.912, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-42de39f1e3e7", + "path_leak": true, + "required_files": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "found": [ + "src/scripts/merge_founder_research.mjs" + ], + "missed": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "recall_at_5": 0.125, + "precision_at_5": 0.2, + "recall_at_10": 0.125, + "precision_at_10": 0.1, + "recall_at_20": 0.125, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.125, + "recall_at_4000_tokens": 0.125, + "recall_at_16000_tokens": 0.125, + "first_hit_rank": 3, + "tokens_delivered": 730, + "latency_ms": 1238.762, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-62d014f029c5", + "path_leak": true, + "required_files": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/build_dataset.mjs" + ], + "found": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/build_dataset.mjs" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 514, + "latency_ms": 555.402, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-6e229e791c6c", + "path_leak": true, + "required_files": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "found": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 877, + "latency_ms": 569.025, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-77787b7d957c", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [], + "missed": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39, + "latency_ms": 503.834, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-8105b178afd6", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "found": [], + "missed": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 674, + "latency_ms": 576.224, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-9d6998dcea66", + "path_leak": true, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 706, + "latency_ms": 617.309, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-a6e68b3385a5", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [], + "missed": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 379.948, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-c699da18123d", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro" + ], + "found": [], + "missed": [ + "src/layouts/Base.astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2895, + "latency_ms": 542.13, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-c9ac67e7bf08", + "path_leak": true, + "required_files": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "found": [ + "src/data/people.csv", + "src/data/people.json" + ], + "missed": [ + "data/research/merge_report.json", + "src/scripts/merge_research.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2857, + "recall_at_20": 0.5, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2647, + "latency_ms": 162.305, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-d3747e8cea4f", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [ + "src/scripts/marble-world.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 838, + "latency_ms": 2092.926, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-dbe87b68b695", + "path_leak": true, + "required_files": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "found": [], + "missed": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 732.119, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-ec48ef20dcb2", + "path_leak": true, + "required_files": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "found": [ + "src/lib/coverage.ts", + "src/scripts/audit_clarity.mjs" + ], + "missed": [ + "src/pages/coverage/index.astro" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 5, + "tokens_delivered": 806, + "latency_ms": 1440.033, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "what-it-takes-to-win-eca794d585ba", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [ + "src/scripts/marble-world.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 709, + "latency_ms": 1375.129, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-3dd1b2f11b56", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2518, + "latency_ms": 2506.805, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-42de39f1e3e7", + "path_leak": true, + "required_files": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "found": [], + "missed": [ + "src/scripts/apply_3d_scores.mjs", + "src/scripts/apply_scoring.mjs", + "src/scripts/build_scoring_batches.mjs", + "src/scripts/build_yc_founder_batches.mjs", + "src/scripts/fetch_yc_companies.mjs", + "src/scripts/heuristic_score.mjs", + "src/scripts/merge_founder_research.mjs", + "src/scripts/scrape_yc_founders.mjs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4162, + "latency_ms": 4270.802, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-62d014f029c5", + "path_leak": true, + "required_files": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/build_dataset.mjs" + ], + "found": [ + "src/scripts/build_dataset.mjs" + ], + "missed": [ + "src/data/archetypes.json", + "src/data/people.csv", + "src/data/people.json" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1111, + "recall_at_20": 0.25, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 5451, + "latency_ms": 1818.913, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-6e229e791c6c", + "path_leak": true, + "required_files": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "found": [ + "src/scripts/audit_discovery.mjs", + "src/scripts/generate_discovery_surfaces.mjs" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 5059, + "latency_ms": 3160.813, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-77787b7d957c", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [], + "missed": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 784, + "latency_ms": 1925.719, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-8105b178afd6", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "found": [ + "src/layouts/Base.astro", + "src/pages/explore/index.astro", + "src/pages/index.astro" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1697, + "latency_ms": 2394.766, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-9d6998dcea66", + "path_leak": true, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 195, + "latency_ms": 2359.108, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-a6e68b3385a5", + "path_leak": false, + "required_files": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "found": [], + "missed": [ + "src/pages/compare/index.astro", + "src/pages/person/[id].astro" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7, + "latency_ms": 1716.235, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-c699da18123d", + "path_leak": false, + "required_files": [ + "src/layouts/Base.astro" + ], + "found": [ + "src/layouts/Base.astro" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5197, + "latency_ms": 2355.442, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-c9ac67e7bf08", + "path_leak": true, + "required_files": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "found": [], + "missed": [ + "data/research/merge_report.json", + "src/data/people.csv", + "src/data/people.json", + "src/scripts/merge_research.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2973, + "latency_ms": 736.375, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-d3747e8cea4f", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [ + "src/scripts/marble-world.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 6591, + "latency_ms": 4660.961, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-dbe87b68b695", + "path_leak": true, + "required_files": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "found": [], + "missed": [ + "public/data/people.json", + "public/data/search-index.json", + "src/data/people.csv", + "src/data/people.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5363, + "latency_ms": 1995.987, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-ec48ef20dcb2", + "path_leak": true, + "required_files": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro", + "src/scripts/audit_clarity.mjs" + ], + "found": [ + "src/lib/coverage.ts", + "src/pages/coverage/index.astro" + ], + "missed": [ + "src/scripts/audit_clarity.mjs" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 951, + "latency_ms": 6436.596, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "what-it-takes-to-win-eca794d585ba", + "path_leak": true, + "required_files": [ + "src/scripts/marble-world.ts" + ], + "found": [ + "src/scripts/marble-world.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 190, + "latency_ms": 4680.92, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/EXCLUDED-code-graph-context-stale-db.json b/benchmarks/context-retrieval/results/full-field-got/EXCLUDED-code-graph-context-stale-db.json new file mode 100644 index 00000000..26312759 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/EXCLUDED-code-graph-context-stale-db.json @@ -0,0 +1,3088 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [ + { + "provider_id": "code-graph-context", + "why": "scored exactly zero — read the raw payload before believing it" + } + ], + "nominated_for_hand_audit": [], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "code-graph-context", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 93, + "median_latency_ms": 32612.1445, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 108, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 93, + "median_latency_ms": 32612.1445, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 93, + "median_latency_ms": 32251.179, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 93, + "median_latency_ms": 33866.2845, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "indexed-but-returned-nothing": 108 + }, + "resources": { + "baseline_rss_mb": 28214.5, + "peak_rss_mb": 43249.7, + "delta_rss_mb": 15035.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "code-graph-context", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 38961.716, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 67834.101, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 59707.635, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 75378.213, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 50849.326, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 68357.588, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 48456.564, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 39129.082, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 163, + "latency_ms": 58953.144, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 44900.328, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 59810.006, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 43635.807, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 22024.313, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 19531.121, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 45865.926, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 46698.465, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 38172.971, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 39817.186, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 47009.011, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 45157.049, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 44095.416, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 19670.928, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 45835.271, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 40414.707, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 48062.522, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 45808.393, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 37957.69, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 40207.61, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 39186.516, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 17337.713, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 35262.113, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 27710.421, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 17237.202, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 44014.9, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 28515.073, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 47463.46, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 42563.361, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 18086.139, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 14230.863, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 46140.588, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 29691.007, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 163, + "latency_ms": 44301.616, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 41177.972, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 13412.952, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 15404.107, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 34101.95, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 35461.846, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 163, + "latency_ms": 37834.014, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 40582.145, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 31162.88, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 39510.742, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 19748.005, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 49857.54, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 25232.813, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 36358.151, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 16117.806, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 14922.003, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 22891.515, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 13830.399, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 16512.2, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 15282.503, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 12387.444, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 11860.533, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 31623.267, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 14579.452, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 31378.633, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 26777.685, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 24073.271, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 13397.481, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 29438.182, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 25961.954, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 30454.005, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 33630.619, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 20540.191, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 12285.906, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 40722.615, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 32291.841, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 41042.772, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 29753.235, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 26964.593, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 32489.484, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 31220.935, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 29481.853, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 32012.874, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 12417.472, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 27148.448, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 163, + "latency_ms": 33240.354, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 163, + "latency_ms": 34270.187, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 29937.798, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93, + "latency_ms": 12961.749, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 14308.896, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 163, + "latency_ms": 33609.681, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 163, + "latency_ms": 37792.526, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 30770.908, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 12458.119, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 186, + "latency_ms": 38218.347, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 28992.749, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 12347.085, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 163, + "latency_ms": 34922.527, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 186, + "latency_ms": 34377.212, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 30187.609, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 29293.788, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 35918.974, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 31165.102, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70, + "latency_ms": 32734.805, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 40037.404, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117, + "latency_ms": 36594.876, + "indexed_revision_matches": true + }, + { + "provider_id": "code-graph-context", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 140, + "latency_ms": 15035.453, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/EXCLUDED-seagoat.json b/benchmarks/context-retrieval/results/full-field-got/EXCLUDED-seagoat.json new file mode 100644 index 00000000..70b131c5 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/EXCLUDED-seagoat.json @@ -0,0 +1,3088 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [ + { + "provider_id": "seagoat", + "why": "scored exactly zero — read the raw payload before believing it" + } + ], + "nominated_for_hand_audit": [], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "seagoat", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 861.934, + "payload_kind": [ + "chunks" + ], + "unavailable": 108 + }, + "baseline_missed": { + "cases": 108, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 861.934, + "payload_kind": [ + "chunks" + ], + "unavailable": 108 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 822.116, + "payload_kind": [ + "chunks" + ], + "unavailable": 54 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 922.869, + "payload_kind": [ + "chunks" + ], + "unavailable": 54 + } + }, + "outcomes": { + "indexed-but-returned-nothing": 108 + }, + "resources": { + "baseline_rss_mb": 20493, + "peak_rss_mb": 28573.3, + "delta_rss_mb": 8080.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "seagoat", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1898.917, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1419.845, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 835.471, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1263.397, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 913.954, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 788.688, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1100.478, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1239.593, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1033.342, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 776.034, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 782.061, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 645.814, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 698.606, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 805.716, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 777.906, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 982.418, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1299.426, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1148.544, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1353.208, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1182.097, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 977.795, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 784.059, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 833.255, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 789.575, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 775.768, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 793.358, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 823.187, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1028.826, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1119.241, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1024.032, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 820.445, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 639.249, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 652.667, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 683.703, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2077.746, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 858.622, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 915.387, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1093.791, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1221.504, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1470.734, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1242.442, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1093.473, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 939.586, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 735.169, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 976.105, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 908.019, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 808.338, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 970.939, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1186.689, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1390.963, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1098.348, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 781.653, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1024.321, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 943.121, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1244.737, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1464.276, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2365.132, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1657.889, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1062.838, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 937.719, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 758.051, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1531.875, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1032.71, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1068.675, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 819.227, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1177.01, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 800.769, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 711.66, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 819.939, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 706.468, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 821.045, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 841.556, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1150.303, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 746.215, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 982.177, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 888.154, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 889.529, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 844.855, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 865.246, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 988.423, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1027.619, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 954.379, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 894.661, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 726.091, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 676.292, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 564.313, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 626.086, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 562.098, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 608.331, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 650.401, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 771.039, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 720.362, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 812.849, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 640.967, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 812.216, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 624.814, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 632.552, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 538.926, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 603.584, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 769.939, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1351.219, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1100.308, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 697.408, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 658.557, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 636.861, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 735.839, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 876.75, + "indexed_revision_matches": true + }, + { + "provider_id": "seagoat", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 767.22, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/agent-default.json b/benchmarks/context-retrieval/results/full-field-got/agent-default.json new file mode 100644 index 00000000..1755fda2 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/agent-default.json @@ -0,0 +1,3134 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "agent-default", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "test/cache.ts", + "test/timeout.ts", + "package.json", + "source/create.ts", + "source/index.ts" + ] + }, + { + "provider_id": "agent-default", + "query": "Fix timeout error firing even though it's cancelled (#2399)", + "revision": "f997bcb385d4b35ff581333e6bc9360d5ee10e71", + "returned": [ + "test/error.ts", + "test/timeout.ts", + "documentation/6-timeout.md", + "test/abort.ts", + "test/cache.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "agent-default", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.4616, + "full_recall_at_5": 0.287, + "mean_recall_at_10": 0.629, + "full_recall_at_10": 0.4537, + "mean_recall_at_20": 0.8177, + "full_recall_at_20": 0.6944, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1543, + "mean_recall_at_16000_tokens": 0.4381, + "mean_precision_at_10": 0.1301, + "zero_hit_rate": 0.0833, + "median_tokens_delivered": 59535.5, + "median_latency_ms": 82.68, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 59, + "mean_recall_at_5": 0.1966, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.3209, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.6664, + "full_recall_at_20": 0.4407, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0537, + "mean_recall_at_16000_tokens": 0.1876, + "mean_precision_at_10": 0.0831, + "zero_hit_rate": 0.1525, + "median_tokens_delivered": 59733, + "median_latency_ms": 81.622, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 49, + "mean_recall_at_5": 0.7806, + "full_recall_at_5": 0.6327, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2755, + "mean_recall_at_16000_tokens": 0.7398, + "mean_precision_at_10": 0.1867, + "zero_hit_rate": 0, + "median_tokens_delivered": 57377, + "median_latency_ms": 82.825, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.6623, + "full_recall_at_5": 0.4444, + "mean_recall_at_10": 0.7765, + "full_recall_at_10": 0.6296, + "mean_recall_at_20": 0.9117, + "full_recall_at_20": 0.7963, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2809, + "mean_recall_at_16000_tokens": 0.6185, + "mean_precision_at_10": 0.1556, + "zero_hit_rate": 0, + "median_tokens_delivered": 60910, + "median_latency_ms": 86.5885, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.2608, + "full_recall_at_5": 0.1296, + "mean_recall_at_10": 0.4815, + "full_recall_at_10": 0.2778, + "mean_recall_at_20": 0.7238, + "full_recall_at_20": 0.5926, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0278, + "mean_recall_at_16000_tokens": 0.2577, + "mean_precision_at_10": 0.1046, + "zero_hit_rate": 0.1667, + "median_tokens_delivered": 58581.5, + "median_latency_ms": 76.715, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 16918.4, + "peak_rss_mb": 18759.1, + "delta_rss_mb": 1840.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "agent-default", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 52061, + "latency_ms": 92.553, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 66384, + "latency_ms": 104.572, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49834, + "latency_ms": 104.323, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53804, + "latency_ms": 64.57, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 68497, + "latency_ms": 131.492, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 141521, + "latency_ms": 108.414, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 52600, + "latency_ms": 91.531, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 58542, + "latency_ms": 83.025, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 87223, + "latency_ms": 108.497, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 45358, + "latency_ms": 61.957, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 102374, + "latency_ms": 82.501, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 46089, + "latency_ms": 75.383, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/normalize-arguments.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 36976, + "latency_ms": 69.073, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 47403, + "latency_ms": 54.563, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 64066, + "latency_ms": 103.786, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 68337, + "latency_ms": 105.737, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 62583, + "latency_ms": 93.077, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 65691, + "latency_ms": 106.401, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 52149, + "latency_ms": 100.414, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 45399, + "latency_ms": 65.14, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 73783, + "latency_ms": 73.891, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 40247, + "latency_ms": 85.727, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 80279, + "latency_ms": 83.638, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 43167, + "latency_ms": 57.089, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 72541, + "latency_ms": 73.317, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 46583, + "latency_ms": 67.376, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 60881, + "latency_ms": 70.576, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 41278, + "latency_ms": 61.005, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 50933, + "latency_ms": 73.662, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 59733, + "latency_ms": 79.932, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 46609, + "latency_ms": 52.965, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 59711, + "latency_ms": 62.848, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 37682, + "latency_ms": 58.214, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 96333, + "latency_ms": 79.205, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62246, + "latency_ms": 52.796, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 83731, + "latency_ms": 86.683, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 76018, + "latency_ms": 86.494, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 41266, + "latency_ms": 94.413, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [ + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 4, + "tokens_delivered": 23953, + "latency_ms": 51.472, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 143218, + "latency_ms": 73.544, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 68236, + "latency_ms": 64.583, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 79989, + "latency_ms": 109.06, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/utils/timer.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 88806, + "latency_ms": 61.061, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 28709, + "latency_ms": 63.114, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 31781, + "latency_ms": 75.35, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47762, + "latency_ms": 69.619, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 78960, + "latency_ms": 82.825, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 86165, + "latency_ms": 95.434, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 64686, + "latency_ms": 101.302, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 59030, + "latency_ms": 54.622, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 80690, + "latency_ms": 77.437, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 7, + "tokens_delivered": 43054, + "latency_ms": 88.55, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 137075, + "latency_ms": 82.763, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 50342, + "latency_ms": 58.688, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 72953, + "latency_ms": 82.597, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 58669, + "latency_ms": 80.74, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 44011, + "latency_ms": 79.26, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 43641, + "latency_ms": 51.263, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 30837, + "latency_ms": 65.468, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 63274, + "latency_ms": 75.993, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 49650, + "latency_ms": 69.265, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 44256, + "latency_ms": 57.853, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 36292, + "latency_ms": 69.768, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 65799, + "latency_ms": 80.528, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 42752, + "latency_ms": 113.365, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 82143, + "latency_ms": 75.444, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [ + "test/progress.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 42213, + "latency_ms": 90.314, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 42655, + "latency_ms": 71.82, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 49037, + "latency_ms": 144.737, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 57803, + "latency_ms": 179.804, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 69811, + "latency_ms": 132.46, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 77408, + "latency_ms": 114.317, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 67101, + "latency_ms": 107.116, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 62026, + "latency_ms": 90.63, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 32327, + "latency_ms": 77.654, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 140781, + "latency_ms": 123.977, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48778, + "latency_ms": 69.872, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 138581, + "latency_ms": 80.288, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 53054, + "latency_ms": 73.895, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28401, + "latency_ms": 66.811, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 84592, + "latency_ms": 84.667, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30034, + "latency_ms": 97.752, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 44953, + "latency_ms": 116.375, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 53717, + "latency_ms": 225.564, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 44547, + "latency_ms": 92.417, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 65288, + "latency_ms": 81.622, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 69847, + "latency_ms": 126.89, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 76641, + "latency_ms": 143.291, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 61652, + "latency_ms": 70.001, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 33300, + "latency_ms": 85.896, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 44051, + "latency_ms": 107.974, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 62628, + "latency_ms": 121.676, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 68002, + "latency_ms": 123.833, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33797, + "latency_ms": 50.397, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 50056, + "latency_ms": 90.636, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 72624, + "latency_ms": 159.73, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 59360, + "latency_ms": 103.778, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 37166, + "latency_ms": 249.254, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 82575, + "latency_ms": 191.838, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 64974, + "latency_ms": 163.571, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 68966, + "latency_ms": 66.782, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37763, + "latency_ms": 64.637, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 90579, + "latency_ms": 94.053, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 57377, + "latency_ms": 76.158, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 60939, + "latency_ms": 67.729, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 84583, + "latency_ms": 99.581, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 86911, + "latency_ms": 121.667, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 64172, + "latency_ms": 110.522, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/ast-grep.json b/benchmarks/context-retrieval/results/full-field-got/ast-grep.json new file mode 100644 index 00000000..ea09bde6 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/ast-grep.json @@ -0,0 +1,3153 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "ast-grep", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "test/cache.ts", + "test/cancel.ts", + "source/core/index.ts", + "readme.md", + "source/types.ts" + ] + }, + { + "provider_id": "ast-grep", + "query": "Fix for sending files with size `0` on `stat` (#1488)", + "revision": "6aa86f2494194907f6c6c8b4774dfa1f69df6876", + "returned": [ + "test/headers.ts", + "test/progress.ts", + "source/core/utils/get-body-size.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "ast-grep", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.2269, + "full_recall_at_5": 0.1019, + "mean_recall_at_10": 0.311, + "full_recall_at_10": 0.1296, + "mean_recall_at_20": 0.434, + "full_recall_at_20": 0.2315, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0463, + "mean_recall_at_16000_tokens": 0.2076, + "mean_precision_at_10": 0.1065, + "zero_hit_rate": 0.3611, + "median_tokens_delivered": 33511.5, + "median_latency_ms": 271.409, + "payload_kind": [ + "whole-files" + ], + "unavailable": 13 + }, + "baseline_missed": { + "cases": 94, + "mean_recall_at_5": 0.1436, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2083, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3496, + "full_recall_at_20": 0.117, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0213, + "mean_recall_at_16000_tokens": 0.1321, + "mean_precision_at_10": 0.092, + "zero_hit_rate": 0.4149, + "median_tokens_delivered": 32661.5, + "median_latency_ms": 277.7225, + "payload_kind": [ + "whole-files" + ], + "unavailable": 13 + }, + "baseline_solved": { + "cases": 14, + "mean_recall_at_5": 0.7857, + "full_recall_at_5": 0.7857, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2143, + "mean_recall_at_16000_tokens": 0.7143, + "mean_precision_at_10": 0.2036, + "zero_hit_rate": 0, + "median_tokens_delivered": 46924, + "median_latency_ms": 240.2915, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3272, + "full_recall_at_5": 0.1852, + "mean_recall_at_10": 0.4383, + "full_recall_at_10": 0.2407, + "mean_recall_at_20": 0.5932, + "full_recall_at_20": 0.3519, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0833, + "mean_recall_at_16000_tokens": 0.3333, + "mean_precision_at_10": 0.119, + "zero_hit_rate": 0.1852, + "median_tokens_delivered": 35804, + "median_latency_ms": 261.5715, + "payload_kind": [ + "whole-files" + ], + "unavailable": 1 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.1265, + "full_recall_at_5": 0.0185, + "mean_recall_at_10": 0.1836, + "full_recall_at_10": 0.0185, + "mean_recall_at_20": 0.2747, + "full_recall_at_20": 0.1111, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0093, + "mean_recall_at_16000_tokens": 0.0818, + "mean_precision_at_10": 0.094, + "zero_hit_rate": 0.537, + "median_tokens_delivered": 25100, + "median_latency_ms": 275.905, + "payload_kind": [ + "whole-files" + ], + "unavailable": 12 + } + }, + "outcomes": { + "answered": 95, + "indexed-but-returned-nothing": 13 + }, + "resources": { + "baseline_rss_mb": 38953, + "peak_rss_mb": 39489.2, + "delta_rss_mb": 536.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "ast-grep", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 66162, + "latency_ms": 334.978, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31614, + "latency_ms": 362.449, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 41985, + "latency_ms": 283.695, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3413, + "latency_ms": 278.115, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 31419, + "latency_ms": 325.069, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/gzip.ts" + ], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 41576, + "latency_ms": 458.485, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35060, + "latency_ms": 262.765, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 28988, + "latency_ms": 277.33, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12904, + "latency_ms": 309.33, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 371.661, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 20714, + "latency_ms": 234.15, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 36548, + "latency_ms": 234.422, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 10, + "tokens_delivered": 24279, + "latency_ms": 236.299, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 31666, + "latency_ms": 167.155, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24307, + "latency_ms": 260.378, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 74294, + "latency_ms": 378.458, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34716, + "latency_ms": 355.77, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 409.934, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 32864, + "latency_ms": 313.481, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 335, + "latency_ms": 2013.98, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4309, + "latency_ms": 595.533, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21051, + "latency_ms": 1034.225, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2491, + "latency_ms": 381.97, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 34175, + "latency_ms": 314.957, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 54311, + "latency_ms": 267.682, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 256.729, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 15467, + "latency_ms": 284.248, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 302.208, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 47782, + "latency_ms": 259.609, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32521, + "latency_ms": 607.704, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 368.847, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 46526, + "latency_ms": 343.343, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 43066, + "latency_ms": 256.848, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 422.71, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 267.802, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 46032, + "latency_ms": 554.181, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 45092, + "latency_ms": 381.125, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 34514, + "latency_ms": 250.667, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 337.88, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.125, + "recall_at_20": 0.3333, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 54260, + "latency_ms": 466.348, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 47127, + "latency_ms": 165.622, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46815, + "latency_ms": 278.964, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25921, + "latency_ms": 180.412, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 142.034, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22237, + "latency_ms": 252.502, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18152, + "latency_ms": 188.586, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 51915, + "latency_ms": 189.54, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.5, + "recall_at_10": 0.6667, + "precision_at_10": 0.5, + "recall_at_20": 0.6667, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 33040, + "latency_ms": 251.052, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 73039, + "latency_ms": 214.768, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 607, + "latency_ms": 183.116, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 41884, + "latency_ms": 273.377, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "test/http.ts", + "test/promise.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 9, + "tokens_delivered": 34350, + "latency_ms": 301.26, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 24897, + "latency_ms": 296.765, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48244, + "latency_ms": 269.441, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 34729, + "latency_ms": 329.124, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3408, + "latency_ms": 346.102, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 49241, + "latency_ms": 253.608, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29514, + "latency_ms": 170.257, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 29706, + "latency_ms": 217.267, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 37118, + "latency_ms": 273.695, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 324.069, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 42019, + "latency_ms": 350.221, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19183, + "latency_ms": 192.567, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 14896, + "latency_ms": 202.83, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 43046, + "latency_ms": 293.251, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 31368, + "latency_ms": 291.725, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33665, + "latency_ms": 378.536, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 29503, + "latency_ms": 167.092, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 40957, + "latency_ms": 203.753, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41683, + "latency_ms": 195.956, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 57037, + "latency_ms": 284.917, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 10784, + "latency_ms": 280.405, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "test/post.ts" + ], + "missed": [ + "source/core/utils/get-body-size.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 62152, + "latency_ms": 234.919, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 69917, + "latency_ms": 288.196, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 34939, + "latency_ms": 329.994, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 67557, + "latency_ms": 417.284, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41497, + "latency_ms": 183.738, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 129843, + "latency_ms": 286.54, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 33358, + "latency_ms": 997.301, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15109, + "latency_ms": 116.19, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39191, + "latency_ms": 216.673, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32551, + "latency_ms": 147.818, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 55412, + "latency_ms": 149.226, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 41625, + "latency_ms": 245.787, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 33677, + "latency_ms": 159.406, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 34833, + "latency_ms": 165.983, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41779, + "latency_ms": 219.525, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 43853, + "latency_ms": 250.378, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/options.ts", + "test/headers.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 9745, + "latency_ms": 140.998, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 38815, + "latency_ms": 286.137, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 61030, + "latency_ms": 241.219, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 52740, + "latency_ms": 257.023, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 37851, + "latency_ms": 259.351, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 131.403, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 21545, + "latency_ms": 275.44, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 51939, + "latency_ms": 485.937, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 18263, + "latency_ms": 239.364, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 46721, + "latency_ms": 222.272, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1667, + "recall_at_20": 0.3333, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 42910, + "latency_ms": 303.658, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 67904, + "latency_ms": 359.299, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 11105, + "latency_ms": 180.267, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 176.427, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 16170, + "latency_ms": 227.675, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 17599, + "latency_ms": 130.012, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 173.069, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "test/create.ts" + ], + "missed": [ + "source/types.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 32772, + "latency_ms": 340.97, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 43243, + "latency_ms": 245.969, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 29858, + "latency_ms": 286.411, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/chunkhound.json b/benchmarks/context-retrieval/results/full-field-got/chunkhound.json new file mode 100644 index 00000000..a2028622 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/chunkhound.json @@ -0,0 +1,3088 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [ + { + "provider_id": "chunkhound", + "why": "scored exactly zero — read the raw payload before believing it" + } + ], + "nominated_for_hand_audit": [], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "chunkhound", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 73, + "median_latency_ms": 9076.4345, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 108, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 73, + "median_latency_ms": 9076.4345, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 73, + "median_latency_ms": 8943.8725, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 73, + "median_latency_ms": 9297.2595, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "indexed-but-returned-nothing": 108 + }, + "resources": { + "baseline_rss_mb": 23334.2, + "peak_rss_mb": 43251.6, + "delta_rss_mb": 19917.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "chunkhound", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 13500.473, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10935.54, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10374.099, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 11104.321, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 13747.869, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10435.452, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8308.906, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8718.197, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8016.602, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8955.44, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9400.317, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9876.129, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 6931.778, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7750.822, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7645.691, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8520.952, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8620.991, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8288.055, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8499.931, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8845.566, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9811.373, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8628.09, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8432.157, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7155.421, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8944.379, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10408.367, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10825.891, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 13990.723, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 16367.351, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 16573.504, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 14706.15, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 12658.861, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10525.587, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9508.403, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 14620.517, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 12626.376, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 20302.456, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 13538.548, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 14824.897, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10743.643, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8943.366, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7995.925, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8432.961, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9026.541, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 20802.189, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 24511.748, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 22690.925, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 11171.595, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7983.783, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8581.601, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7911.068, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8830.785, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7444.266, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 12871.323, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10567.044, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7606.813, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7010.562, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 6379.815, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 6762.735, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7189.679, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 11537.564, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9296.424, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7543.989, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7462.526, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8462.699, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10144.373, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7598.405, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8730.749, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 6558.09, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 6465.98, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 6353.767, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 6691.133, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7606.287, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9126.328, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7768.284, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 11533.89, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9736.876, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10223.477, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8817.89, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8794.094, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7705.885, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7516.458, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 11681.78, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8468.931, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7919.294, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7850.505, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8495.929, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9905.311, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 13637.931, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10043.292, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9520.688, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9287.68, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 14890.134, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 12164.077, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10007.237, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8794.067, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9298.095, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9150.505, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10218.631, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 11354.984, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7224.657, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8015.772, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9325.06, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 10521.097, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9850.658, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 9629.754, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 8095.639, + "indexed_revision_matches": true + }, + { + "provider_id": "chunkhound", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 7001.79, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/churn-ranked.json b/benchmarks/context-retrieval/results/full-field-got/churn-ranked.json new file mode 100644 index 00000000..0c20e13c --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/churn-ranked.json @@ -0,0 +1,3144 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files" + ], + "reason": "controls absent: random-files, random-code-files. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "churn-ranked", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "readme.md", + "package.json", + "source/core/index.ts", + "source/create.ts", + "test/timeout.ts" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Fix for sending files with size `0` on `stat` (#1488)", + "revision": "6aa86f2494194907f6c6c8b4774dfa1f69df6876", + "returned": [ + "readme.md", + "package.json", + "source/core/index.ts", + "source/create.ts", + "test/timeout.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.392, + "full_recall_at_5": 0.1296, + "mean_recall_at_10": 0.5642, + "full_recall_at_10": 0.3333, + "mean_recall_at_20": 0.7651, + "full_recall_at_20": 0.5926, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.2886, + "mean_precision_at_10": 0.112, + "zero_hit_rate": 0.0833, + "median_tokens_delivered": 86197, + "median_latency_ms": 32.636, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 72, + "mean_recall_at_5": 0.2685, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.3463, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.6477, + "full_recall_at_20": 0.3889, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.2176, + "mean_precision_at_10": 0.0833, + "zero_hit_rate": 0.125, + "median_tokens_delivered": 84035.5, + "median_latency_ms": 32.656, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 36, + "mean_recall_at_5": 0.6389, + "full_recall_at_5": 0.3889, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.4306, + "mean_precision_at_10": 0.1694, + "zero_hit_rate": 0, + "median_tokens_delivered": 87954.5, + "median_latency_ms": 32.488, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3025, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.4648, + "full_recall_at_10": 0.1852, + "mean_recall_at_20": 0.7062, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.2191, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.0926, + "median_tokens_delivered": 86197, + "median_latency_ms": 32.6335, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.4815, + "full_recall_at_5": 0.2222, + "mean_recall_at_10": 0.6636, + "full_recall_at_10": 0.4815, + "mean_recall_at_20": 0.8241, + "full_recall_at_20": 0.6852, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.358, + "mean_precision_at_10": 0.1241, + "zero_hit_rate": 0.0741, + "median_tokens_delivered": 85194.5, + "median_latency_ms": 32.636, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 16796.2, + "peak_rss_mb": 17226.8, + "delta_rss_mb": 430.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "churn-ranked", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 75728, + "latency_ms": 57.531, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 99802, + "latency_ms": 34.315, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 90822, + "latency_ms": 36.263, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 77896, + "latency_ms": 50.486, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 79784, + "latency_ms": 37.61, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/gzip.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 172965, + "latency_ms": 32.913, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 97706, + "latency_ms": 38.48, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 77975, + "latency_ms": 40.388, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 114321, + "latency_ms": 41.95, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 76235, + "latency_ms": 34.522, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 173145, + "latency_ms": 44.779, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 76542, + "latency_ms": 60.637, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 65911, + "latency_ms": 39.356, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 88588, + "latency_ms": 38.083, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 78077, + "latency_ms": 35.019, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 102025, + "latency_ms": 35.946, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 77315, + "latency_ms": 32.303, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 76092, + "latency_ms": 31.106, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 101214, + "latency_ms": 36.892, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 82251, + "latency_ms": 36.961, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 97035, + "latency_ms": 34.721, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 70318, + "latency_ms": 35.212, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 101069, + "latency_ms": 32.63, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 81001, + "latency_ms": 31.332, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 111568, + "latency_ms": 30.856, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 82727, + "latency_ms": 32.115, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82726, + "latency_ms": 32.052, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 97778, + "latency_ms": 39.505, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82458, + "latency_ms": 34.045, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 87827, + "latency_ms": 37.671, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 76187, + "latency_ms": 32.67, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 90825, + "latency_ms": 34.518, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 67660, + "latency_ms": 30.267, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 114374, + "latency_ms": 31.442, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 91002, + "latency_ms": 32.383, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [ + "source/types.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 114575, + "latency_ms": 33.2, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 97501, + "latency_ms": 37.396, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 71352, + "latency_ms": 36.779, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 66233, + "latency_ms": 33.311, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 173739, + "latency_ms": 32.302, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 76451, + "latency_ms": 32.959, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 99234, + "latency_ms": 30.508, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 114374, + "latency_ms": 29.275, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 74089, + "latency_ms": 33.951, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 73631, + "latency_ms": 35.136, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 82954, + "latency_ms": 35.054, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 93966, + "latency_ms": 35.555, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 99459, + "latency_ms": 29.084, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 100118, + "latency_ms": 32.67, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 77852, + "latency_ms": 32.275, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 91396, + "latency_ms": 28.622, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts" + ], + "missed": [ + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 71360, + "latency_ms": 32.642, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 173169, + "latency_ms": 30.575, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 75153, + "latency_ms": 33.631, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 90913, + "latency_ms": 35.576, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/utils/get-body-size.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 89127, + "latency_ms": 31.79, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 65701, + "latency_ms": 29.012, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 74440, + "latency_ms": 29.647, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 67635, + "latency_ms": 27.752, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 88624, + "latency_ms": 27.27, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 71858, + "latency_ms": 34.421, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 72200, + "latency_ms": 34.719, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 67311, + "latency_ms": 36.883, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 91043, + "latency_ms": 38.164, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 71278, + "latency_ms": 34.54, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 104823, + "latency_ms": 32.346, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 79450, + "latency_ms": 30.475, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 76180, + "latency_ms": 27.631, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 75699, + "latency_ms": 28.098, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 90992, + "latency_ms": 26.43, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 77091, + "latency_ms": 29.404, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 94655, + "latency_ms": 30.119, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 95014, + "latency_ms": 31.163, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 74341, + "latency_ms": 37.346, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 73276, + "latency_ms": 36.452, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [ + "test/early-307-redirects.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 173964, + "latency_ms": 28.173, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 83068, + "latency_ms": 26.045, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 171546, + "latency_ms": 25.53, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 81558, + "latency_ms": 24.245, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 79625, + "latency_ms": 26.982, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 105263, + "latency_ms": 28.131, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 91110, + "latency_ms": 30.109, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 81691, + "latency_ms": 33.773, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 85344, + "latency_ms": 34.052, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 88604, + "latency_ms": 31.151, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 77136, + "latency_ms": 29.034, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 91019, + "latency_ms": 28.591, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 93273, + "latency_ms": 28.502, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 76362, + "latency_ms": 31.665, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 70358, + "latency_ms": 31.495, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 73084, + "latency_ms": 32.597, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 90795, + "latency_ms": 32.674, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 91132, + "latency_ms": 37.022, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 87321, + "latency_ms": 33.704, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 74546, + "latency_ms": 26.704, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 99960, + "latency_ms": 28.827, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 76867, + "latency_ms": 29.937, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 73178, + "latency_ms": 30.597, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 101498, + "latency_ms": 29.626, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 92921, + "latency_ms": 31.961, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 79914, + "latency_ms": 30.855, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 78848, + "latency_ms": 33.984, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 107593, + "latency_ms": 31.283, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 90992, + "latency_ms": 27.1, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "source/create.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 87050, + "latency_ms": 30.781, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "test/hooks.ts", + "test/timeout.ts" + ], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 114530, + "latency_ms": 35.116, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 97221, + "latency_ms": 43.31, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 90109, + "latency_ms": 34.338, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/ck.json b/benchmarks/context-retrieval/results/full-field-got/ck.json new file mode 100644 index 00000000..3314333b --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/ck.json @@ -0,0 +1,3145 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "ck", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "media/logo.ai", + "readme.md", + "source/core/index.ts", + "test/http.ts", + "test/cache.ts" + ] + }, + { + "provider_id": "ck", + "query": "Fix timeout error firing even though it's cancelled (#2399)", + "revision": "f997bcb385d4b35ff581333e6bc9360d5ee10e71", + "returned": [ + "media/logo.ai", + "test/timeout.ts", + "source/core/timed-out.ts", + "documentation/7-retry.md", + "documentation/async-stack-traces.md" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "ck", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.529, + "full_recall_at_5": 0.3241, + "mean_recall_at_10": 0.7279, + "full_recall_at_10": 0.5463, + "mean_recall_at_20": 0.7488, + "full_recall_at_20": 0.5741, + "mean_recall_at_1000_tokens": 0.3551, + "mean_recall_at_4000_tokens": 0.7488, + "mean_recall_at_16000_tokens": 0.7488, + "mean_precision_at_10": 0.167, + "zero_hit_rate": 0.1019, + "median_tokens_delivered": 2610, + "median_latency_ms": 34850.259, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 49, + "mean_recall_at_5": 0.2986, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.4003, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.4463, + "full_recall_at_20": 0.0612, + "mean_recall_at_1000_tokens": 0.2214, + "mean_recall_at_4000_tokens": 0.4463, + "mean_recall_at_16000_tokens": 0.4463, + "mean_precision_at_10": 0.1237, + "zero_hit_rate": 0.2245, + "median_tokens_delivered": 2593, + "median_latency_ms": 34371.02, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 59, + "mean_recall_at_5": 0.7203, + "full_recall_at_5": 0.5932, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.4661, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.2028, + "zero_hit_rate": 0, + "median_tokens_delivered": 2612, + "median_latency_ms": 36392.292, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.5549, + "full_recall_at_5": 0.3519, + "mean_recall_at_10": 0.7383, + "full_recall_at_10": 0.5556, + "mean_recall_at_20": 0.7568, + "full_recall_at_20": 0.5741, + "mean_recall_at_1000_tokens": 0.3475, + "mean_recall_at_4000_tokens": 0.7568, + "mean_recall_at_16000_tokens": 0.7568, + "mean_precision_at_10": 0.1774, + "zero_hit_rate": 0.0926, + "median_tokens_delivered": 2607, + "median_latency_ms": 33929.5085, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.5031, + "full_recall_at_5": 0.2963, + "mean_recall_at_10": 0.7176, + "full_recall_at_10": 0.537, + "mean_recall_at_20": 0.7407, + "full_recall_at_20": 0.5741, + "mean_recall_at_1000_tokens": 0.3627, + "mean_recall_at_4000_tokens": 0.7407, + "mean_recall_at_16000_tokens": 0.7407, + "mean_precision_at_10": 0.1565, + "zero_hit_rate": 0.1111, + "median_tokens_delivered": 2612.5, + "median_latency_ms": 35880.1865, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 18998.4, + "peak_rss_mb": 43285.2, + "delta_rss_mb": 24286.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "ck", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2512, + "latency_ms": 63476.418, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2649, + "latency_ms": 109948.489, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 2517, + "latency_ms": 197199.084, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2629, + "latency_ms": 89972.603, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 2688, + "latency_ms": 89974.517, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/http.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2726, + "latency_ms": 244069.006, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2594, + "latency_ms": 150506.796, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2478, + "latency_ms": 87895.967, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2812, + "latency_ms": 118443.012, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2648, + "latency_ms": 83023.161, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2532, + "latency_ms": 207977.931, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2646, + "latency_ms": 60839.063, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.125, + "recall_at_20": 0.3333, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 2557, + "latency_ms": 41733.913, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2612, + "latency_ms": 59885.08, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2677, + "latency_ms": 58985.219, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2507, + "latency_ms": 79141.826, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2555, + "latency_ms": 57104.361, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 2743, + "latency_ms": 61819.098, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2571, + "latency_ms": 71242.169, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2610, + "latency_ms": 58464.129, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2709, + "latency_ms": 63886.989, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 2582, + "latency_ms": 36792.017, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2659, + "latency_ms": 63215.671, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 2521, + "latency_ms": 55220.081, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2753, + "latency_ms": 68273.43, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json", + "tsconfig.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2542, + "latency_ms": 49965.355, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2492, + "latency_ms": 50368.585, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2683, + "latency_ms": 65955.2, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2639, + "latency_ms": 49772.082, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2500, + "latency_ms": 37733.819, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts", + "source/core/response.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2857, + "recall_at_20": 0.5, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2502, + "latency_ms": 44261.398, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2605, + "latency_ms": 39505.059, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2617, + "latency_ms": 36392.292, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.4286, + "recall_at_20": 1, + "precision_at_20": 0.4286, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2634, + "latency_ms": 78508.373, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2647, + "latency_ms": 41004.871, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2727, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 2465, + "latency_ms": 62204.994, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 2559, + "latency_ms": 53893.707, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 2686, + "latency_ms": 31735.441, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [ + "test/agent.ts", + "test/retry.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 2468, + "latency_ms": 26114.097, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.25, + "recall_at_20": 0.6667, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 2636, + "latency_ms": 56213.602, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2483, + "latency_ms": 28432.589, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2668, + "latency_ms": 34327.059, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2739, + "latency_ms": 39593.1, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2552, + "latency_ms": 19157.702, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/core.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 2580, + "latency_ms": 19922.065, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2480, + "latency_ms": 30606.51, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2430, + "latency_ms": 33487.997, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2222, + "recall_at_20": 0.6667, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 2498, + "latency_ms": 34959.894, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 2559, + "latency_ms": 34371.02, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.6, + "precision_at_20": 0.2727, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 3, + "tokens_delivered": 2710, + "latency_ms": 24743.012, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2576, + "latency_ms": 35295.685, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2546, + "latency_ms": 19433.576, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2595, + "latency_ms": 57736.818, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2758, + "latency_ms": 24597.983, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.25, + "recall_at_20": 0.6667, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 2518, + "latency_ms": 32342.713, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2617, + "latency_ms": 24030.218, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 2651, + "latency_ms": 17968.436, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2702, + "latency_ms": 25133.23, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2676, + "latency_ms": 19510.468, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2626, + "latency_ms": 23371.663, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2703, + "latency_ms": 19513.529, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 2691, + "latency_ms": 19087.663, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2541, + "latency_ms": 18730.501, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2511, + "latency_ms": 31523.385, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2588, + "latency_ms": 23410.112, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 2613, + "latency_ms": 36689.703, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2671, + "latency_ms": 30134.498, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2674, + "latency_ms": 29809.154, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2670, + "latency_ms": 21354.374, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 2631, + "latency_ms": 32775.903, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2640, + "latency_ms": 28132.281, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2672, + "latency_ms": 34415.078, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2535, + "latency_ms": 32815.426, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1667, + "recall_at_20": 0.3333, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 2494, + "latency_ms": 25112.459, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2604, + "latency_ms": 20267.154, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3182, + "latency_ms": 54634.848, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 2667, + "latency_ms": 33277.32, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2668, + "latency_ms": 55984.837, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2481, + "latency_ms": 30920.028, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 2790, + "latency_ms": 31027.827, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2610, + "latency_ms": 38573.197, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 2587, + "latency_ms": 33938.67, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2753, + "latency_ms": 28118.243, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2534, + "latency_ms": 35347.19, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 2566, + "latency_ms": 23026.981, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2783, + "latency_ms": 26873.527, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 2737, + "latency_ms": 31153.658, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2544, + "latency_ms": 36824.689, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2222, + "recall_at_20": 0.6667, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 2547, + "latency_ms": 29420.978, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2684, + "latency_ms": 17853.461, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2643, + "latency_ms": 19961.314, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 2686, + "latency_ms": 29359.564, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2679, + "latency_ms": 33164.742, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 2557, + "latency_ms": 33060.06, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 2551, + "latency_ms": 20771.563, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2575, + "latency_ms": 34851.982, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2547, + "latency_ms": 25865.669, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2662, + "latency_ms": 20812.507, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2222, + "recall_at_20": 0.6667, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 7, + "tokens_delivered": 2800, + "latency_ms": 34848.536, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 2555, + "latency_ms": 34775.329, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 2586, + "latency_ms": 29314.985, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2615, + "latency_ms": 27156.294, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2481, + "latency_ms": 41476.742, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2601, + "latency_ms": 36161.158, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2419, + "latency_ms": 35027.075, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/extend.types.ts", + "test/hooks.ts" + ], + "missed": [ + "test/create.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.6, + "precision_at_5": 0.6, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.6, + "precision_at_20": 0.2727, + "recall_at_1000_tokens": 0.4, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 3, + "tokens_delivered": 2593, + "latency_ms": 40746.577, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2545, + "latency_ms": 36464.688, + "indexed_revision_matches": true + }, + { + "provider_id": "ck", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 2636, + "latency_ms": 25869.608, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/cocoindex-code.json b/benchmarks/context-retrieval/results/full-field-got/cocoindex-code.json new file mode 100644 index 00000000..129d209e --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/cocoindex-code.json @@ -0,0 +1,3155 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "cocoindex-code", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "readme.md", + "package.json", + "source/core/index.ts", + "source/index.ts", + "test/types/slow-stream/index.d.ts" + ] + }, + { + "provider_id": "cocoindex-code", + "query": "Fix it not using HTTP/2 connection reuse by default", + "revision": "dc4f1e323dbdc0c4c2dbf1e03c66b714bfb51d39", + "returned": [ + "package.json", + "source/core/options.ts", + "documentation/typescript.md", + "documentation/async-stack-traces.md", + "tsconfig.json" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "cocoindex-code", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.3486, + "full_recall_at_5": 0.1759, + "mean_recall_at_10": 0.5074, + "full_recall_at_10": 0.3148, + "mean_recall_at_20": 0.5185, + "full_recall_at_20": 0.3241, + "mean_recall_at_1000_tokens": 0.0744, + "mean_recall_at_4000_tokens": 0.396, + "mean_recall_at_16000_tokens": 0.5185, + "mean_precision_at_10": 0.122, + "zero_hit_rate": 0.2593, + "median_tokens_delivered": 5007.5, + "median_latency_ms": 1799.8455, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 74, + "mean_recall_at_5": 0.1912, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2811, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2973, + "full_recall_at_20": 0.0135, + "mean_recall_at_1000_tokens": 0.0162, + "mean_recall_at_4000_tokens": 0.2153, + "mean_recall_at_16000_tokens": 0.2973, + "mean_precision_at_10": 0.0842, + "zero_hit_rate": 0.3784, + "median_tokens_delivered": 4997, + "median_latency_ms": 1795.29, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 34, + "mean_recall_at_5": 0.6912, + "full_recall_at_5": 0.5588, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.201, + "mean_recall_at_4000_tokens": 0.7892, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.2041, + "zero_hit_rate": 0, + "median_tokens_delivered": 5028, + "median_latency_ms": 1812.8435, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3531, + "full_recall_at_5": 0.1481, + "mean_recall_at_10": 0.5148, + "full_recall_at_10": 0.3148, + "mean_recall_at_20": 0.537, + "full_recall_at_20": 0.3333, + "mean_recall_at_1000_tokens": 0.0562, + "mean_recall_at_4000_tokens": 0.3969, + "mean_recall_at_16000_tokens": 0.537, + "mean_precision_at_10": 0.1271, + "zero_hit_rate": 0.2407, + "median_tokens_delivered": 5036, + "median_latency_ms": 1816.698, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3441, + "full_recall_at_5": 0.2037, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.3148, + "mean_recall_at_20": 0.5, + "full_recall_at_20": 0.3148, + "mean_recall_at_1000_tokens": 0.0926, + "mean_recall_at_4000_tokens": 0.3951, + "mean_recall_at_16000_tokens": 0.5, + "mean_precision_at_10": 0.1169, + "zero_hit_rate": 0.2778, + "median_tokens_delivered": 4990.5, + "median_latency_ms": 1787.2705, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 17686.7, + "peak_rss_mb": 39472.4, + "delta_rss_mb": 21785.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "cocoindex-code", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 5071, + "latency_ms": 1710.912, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4995, + "latency_ms": 1705.651, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5036, + "latency_ms": 1926.151, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4271, + "latency_ms": 1873.652, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4678, + "latency_ms": 1491.078, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4552, + "latency_ms": 1371.574, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5238, + "latency_ms": 1592.842, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 5244, + "latency_ms": 1656.586, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 5028, + "latency_ms": 2162.253, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 4489, + "latency_ms": 2408.981, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 5049, + "latency_ms": 1782.674, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4596, + "latency_ms": 1891.726, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4802, + "latency_ms": 1381.662, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5137, + "latency_ms": 1917.412, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 4390, + "latency_ms": 1814.663, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5264, + "latency_ms": 1438.07, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4108, + "latency_ms": 1432.042, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 4703, + "latency_ms": 1763.467, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4816, + "latency_ms": 1899.842, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5015, + "latency_ms": 1841.442, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 5355, + "latency_ms": 2957.927, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5355, + "latency_ms": 1693.172, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 5047, + "latency_ms": 1822.651, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5109, + "latency_ms": 1950.827, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5175, + "latency_ms": 1831.905, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json", + "tsconfig.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4804, + "latency_ms": 2671.304, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 5431, + "latency_ms": 1695.076, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 5319, + "latency_ms": 1541.43, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 5453, + "latency_ms": 1784.671, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4142, + "latency_ms": 1637.813, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts", + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 4840, + "latency_ms": 2769.373, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 4342, + "latency_ms": 1714.674, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5021, + "latency_ms": 1731.76, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2727, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5146, + "latency_ms": 1768.892, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5179, + "latency_ms": 1589.647, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/types.ts" + ], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1111, + "recall_at_20": 0.3333, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 4541, + "latency_ms": 1527.403, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 4767, + "latency_ms": 2724.303, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5199, + "latency_ms": 1521.397, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 8, + "tokens_delivered": 4752, + "latency_ms": 1592.254, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1111, + "recall_at_20": 0.3333, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 4510, + "latency_ms": 1588.884, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 4720, + "latency_ms": 13569.144, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5044, + "latency_ms": 1548.093, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 5302, + "latency_ms": 1605.025, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 4596, + "latency_ms": 1845.732, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/core.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2222, + "recall_at_20": 0.6667, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 5155, + "latency_ms": 1735.893, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3510, + "latency_ms": 1805.909, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 5029, + "latency_ms": 1568.725, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1667, + "recall_at_20": 0.3333, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 6, + "tokens_delivered": 5079, + "latency_ms": 1539.917, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "test/retry.ts" + ], + "missed": [ + "source/as-promise/index.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 7, + "tokens_delivered": 5161, + "latency_ms": 1511.671, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/helpers.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/error.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 10, + "tokens_delivered": 4635, + "latency_ms": 1758.834, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 5069, + "latency_ms": 1690.86, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1429, + "recall_at_20": 0.25, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 4, + "tokens_delivered": 4436, + "latency_ms": 3394.939, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 5318, + "latency_ms": 1721.134, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 5141, + "latency_ms": 1673.427, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5265, + "latency_ms": 1591.981, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 5282, + "latency_ms": 1850.747, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 5204, + "latency_ms": 1537.442, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5112, + "latency_ms": 1793.782, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 4734, + "latency_ms": 2041.38, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.25, + "recall_at_20": 0.6667, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 5549, + "latency_ms": 1895.706, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 4691, + "latency_ms": 1787.026, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.125, + "recall_at_20": 0.3333, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 8, + "tokens_delivered": 4848, + "latency_ms": 3914.299, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 5057, + "latency_ms": 1697.413, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 5327, + "latency_ms": 1874.449, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 5191, + "latency_ms": 1751.996, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 4834, + "latency_ms": 4188.798, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5278, + "latency_ms": 4250.796, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 4582, + "latency_ms": 3633.572, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5035, + "latency_ms": 5193.863, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4836, + "latency_ms": 3476.114, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 4748, + "latency_ms": 3050.869, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 4884, + "latency_ms": 2028.99, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4999, + "latency_ms": 2368.865, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2727, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4836, + "latency_ms": 2568.112, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4678, + "latency_ms": 2260.259, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5332, + "latency_ms": 2128.862, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4987, + "latency_ms": 1611.186, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 4501, + "latency_ms": 1762.025, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 5000, + "latency_ms": 2039.388, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 5064, + "latency_ms": 2064.05, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 5105, + "latency_ms": 1995.988, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5166, + "latency_ms": 2133.335, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 4881, + "latency_ms": 2624.816, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5037, + "latency_ms": 1818.733, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 4994, + "latency_ms": 1918.668, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4969, + "latency_ms": 1787.515, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4925, + "latency_ms": 2854.468, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5317, + "latency_ms": 1709.544, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/headers.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1111, + "recall_at_20": 0.3333, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 4442, + "latency_ms": 1705.737, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 4735, + "latency_ms": 1602.803, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4672, + "latency_ms": 1538.708, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 4416, + "latency_ms": 1698.688, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4734, + "latency_ms": 1748.666, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 4923, + "latency_ms": 2123.883, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 5084, + "latency_ms": 1872.616, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 5130, + "latency_ms": 1926.335, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4653, + "latency_ms": 1957.729, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 5125, + "latency_ms": 1684.743, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 4985, + "latency_ms": 1899.224, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4948, + "latency_ms": 1718.677, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5049, + "latency_ms": 1575.278, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5072, + "latency_ms": 1666.062, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4767, + "latency_ms": 1957.814, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 5050, + "latency_ms": 1727.051, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5270, + "latency_ms": 2001.816, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts" + ], + "missed": [ + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.6, + "precision_at_10": 0.3, + "recall_at_20": 0.6, + "precision_at_20": 0.3, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 1, + "tokens_delivered": 4895, + "latency_ms": 2330.773, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 5236, + "latency_ms": 3189.335, + "indexed_revision_matches": true + }, + { + "provider_id": "cocoindex-code", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 4837, + "latency_ms": 3502.893, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/code-review-graph.json b/benchmarks/context-retrieval/results/full-field-got/code-review-graph.json new file mode 100644 index 00000000..e42701e1 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/code-review-graph.json @@ -0,0 +1,3151 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "code-review-graph", + "query": "Fix encoding with json `responseType` (#1996)", + "revision": "dbbd31777b3299ceb0db99798453e01ab8e597b1", + "returned": [ + "test/cache.ts", + "source/core/options.ts", + "test/headers.ts", + "test/http.ts", + "test/arguments.ts" + ] + }, + { + "provider_id": "code-review-graph", + "query": "Fix hanging promise on timeout on HTTP error", + "revision": "7582d916db72f7bab2b2c162d37cfcfde08bbba7", + "returned": [ + "test/error.ts", + "source/core/index.ts", + "test/hooks.ts", + "test/pagination.ts", + "test/timeout.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "code-review-graph", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.3884, + "full_recall_at_5": 0.1944, + "mean_recall_at_10": 0.5557, + "full_recall_at_10": 0.3519, + "mean_recall_at_20": 0.6906, + "full_recall_at_20": 0.4907, + "mean_recall_at_1000_tokens": 0.1636, + "mean_recall_at_4000_tokens": 0.3702, + "mean_recall_at_16000_tokens": 0.669, + "mean_precision_at_10": 0.1147, + "zero_hit_rate": 0.1389, + "median_tokens_delivered": 9620.5, + "median_latency_ms": 3087.3145, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 70, + "mean_recall_at_5": 0.2493, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.3145, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.5226, + "full_recall_at_20": 0.2143, + "mean_recall_at_1000_tokens": 0.0595, + "mean_recall_at_4000_tokens": 0.2069, + "mean_recall_at_16000_tokens": 0.4893, + "mean_precision_at_10": 0.0859, + "zero_hit_rate": 0.2143, + "median_tokens_delivered": 9854.5, + "median_latency_ms": 3060.3335, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 38, + "mean_recall_at_5": 0.6447, + "full_recall_at_5": 0.5526, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.3553, + "mean_recall_at_4000_tokens": 0.6711, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1679, + "zero_hit_rate": 0, + "median_tokens_delivered": 9406.5, + "median_latency_ms": 3169.959, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.4327, + "full_recall_at_5": 0.2037, + "mean_recall_at_10": 0.5975, + "full_recall_at_10": 0.3519, + "mean_recall_at_20": 0.7438, + "full_recall_at_20": 0.5185, + "mean_recall_at_1000_tokens": 0.2191, + "mean_recall_at_4000_tokens": 0.4025, + "mean_recall_at_16000_tokens": 0.7099, + "mean_precision_at_10": 0.1219, + "zero_hit_rate": 0.0556, + "median_tokens_delivered": 11115.5, + "median_latency_ms": 3177.7785, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3441, + "full_recall_at_5": 0.1852, + "mean_recall_at_10": 0.5139, + "full_recall_at_10": 0.3519, + "mean_recall_at_20": 0.6373, + "full_recall_at_20": 0.463, + "mean_recall_at_1000_tokens": 0.108, + "mean_recall_at_4000_tokens": 0.338, + "mean_recall_at_16000_tokens": 0.6281, + "mean_precision_at_10": 0.1076, + "zero_hit_rate": 0.2222, + "median_tokens_delivered": 9193.5, + "median_latency_ms": 2971.4215, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 107, + "indexed-but-returned-nothing": 1 + }, + "resources": { + "baseline_rss_mb": 21849.2, + "peak_rss_mb": 22934.1, + "delta_rss_mb": 1084.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "code-review-graph", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7640, + "latency_ms": 2736.266, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4769, + "latency_ms": 3660.531, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 8938, + "latency_ms": 2961.337, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5058, + "latency_ms": 2954.399, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 17263, + "latency_ms": 2764.108, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 3, + "tokens_delivered": 23648, + "latency_ms": 4771.155, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 9855, + "latency_ms": 3338.001, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4821, + "latency_ms": 3317.08, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 9570, + "latency_ms": 4162.987, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 6395, + "latency_ms": 3591.64, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 9449, + "latency_ms": 5537.741, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4384, + "latency_ms": 2607.173, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/normalize-arguments.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 7603, + "latency_ms": 2070.698, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 8110, + "latency_ms": 2169.842, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 16717, + "latency_ms": 2926.273, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "test/retry.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 9967, + "latency_ms": 3231.973, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 15654, + "latency_ms": 3401.873, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 10910, + "latency_ms": 4338.175, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 13400, + "latency_ms": 3881.184, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1266, + "latency_ms": 3147.313, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 9954, + "latency_ms": 3803.741, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 9671, + "latency_ms": 2950.585, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 16755, + "latency_ms": 4027.755, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4398, + "latency_ms": 2625.096, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 9157, + "latency_ms": 2604.602, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4961, + "latency_ms": 2655.887, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 8607, + "latency_ms": 2684.175, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 5593, + "latency_ms": 2815.482, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 13224, + "latency_ms": 2920.876, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 10114, + "latency_ms": 3249.343, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts", + "source/core/response.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 9057, + "latency_ms": 3190.519, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 5265, + "latency_ms": 2908.674, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 10995, + "latency_ms": 2171.19, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 11, + "tokens_delivered": 13747, + "latency_ms": 2789.072, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1890, + "latency_ms": 2144.381, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts", + "source/types.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 12812, + "latency_ms": 4081.279, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 15104, + "latency_ms": 3776.164, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 12184, + "latency_ms": 3719.847, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 690, + "latency_ms": 2683.263, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 9446, + "latency_ms": 4896.479, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 7049, + "latency_ms": 3090.575, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 22717, + "latency_ms": 3409.411, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 8774, + "latency_ms": 3266.676, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4803, + "latency_ms": 2209.823, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2143, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 13456, + "latency_ms": 2522.817, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4458, + "latency_ms": 2429.926, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 14470, + "latency_ms": 4023.806, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 14152, + "latency_ms": 4696.243, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 13213, + "latency_ms": 4112.457, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 7, + "tokens_delivered": 4516, + "latency_ms": 2841.094, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 13713, + "latency_ms": 2760.663, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 12685, + "latency_ms": 2624.262, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 14454, + "latency_ms": 3865.407, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 6417, + "latency_ms": 2671.97, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 12465, + "latency_ms": 3695.797, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 6307, + "latency_ms": 3797.688, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 12208, + "latency_ms": 2817.067, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 4217, + "latency_ms": 2270.526, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 8668, + "latency_ms": 2266.156, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 8868, + "latency_ms": 2562.621, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 11565, + "latency_ms": 2215.176, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 12, + "tokens_delivered": 12492, + "latency_ms": 2875.854, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 6937, + "latency_ms": 2818.264, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 9364, + "latency_ms": 3381.953, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 16532, + "latency_ms": 2967.911, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 7512, + "latency_ms": 2930.307, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4615, + "latency_ms": 2753.539, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 4222, + "latency_ms": 2554.107, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 9416, + "latency_ms": 2687.65, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6817, + "latency_ms": 3802.273, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 12834, + "latency_ms": 3084.054, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 13389, + "latency_ms": 2981.506, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "test/post.ts" + ], + "missed": [ + "source/core/utils/get-body-size.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 14630, + "latency_ms": 3158.399, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 9, + "tokens_delivered": 13310, + "latency_ms": 2772.868, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 8600, + "latency_ms": 2162.615, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 8, + "tokens_delivered": 26200, + "latency_ms": 4286.238, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 8931, + "latency_ms": 3791.804, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 11236, + "latency_ms": 5239.733, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5445, + "latency_ms": 3477.501, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 344, + "latency_ms": 2841.683, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 12943, + "latency_ms": 2880.973, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 5320, + "latency_ms": 2719.324, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 8237, + "latency_ms": 2356.822, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 6015, + "latency_ms": 3231.832, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 8616, + "latency_ms": 2621.975, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 11606, + "latency_ms": 3744.347, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 24312, + "latency_ms": 4137.579, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 11719, + "latency_ms": 3842.528, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2308, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 7007, + "latency_ms": 3036.613, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 13348, + "latency_ms": 2586.846, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 17711, + "latency_ms": 3421.225, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 21443, + "latency_ms": 3484.868, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 19745, + "latency_ms": 3348.041, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 925, + "latency_ms": 2240.394, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 9755, + "latency_ms": 3463.333, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 32425, + "latency_ms": 3822.988, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 9330, + "latency_ms": 3272.007, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 8502, + "latency_ms": 2292.898, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 10, + "tokens_delivered": 12130, + "latency_ms": 3123.725, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 23138, + "latency_ms": 4516.872, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 11459, + "latency_ms": 3505.649, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6274, + "latency_ms": 3201.103, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 20, + "tokens_delivered": 11288, + "latency_ms": 3933.799, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 5004, + "latency_ms": 3256.529, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 9035, + "latency_ms": 3379.205, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.8, + "first_hit_rank": 2, + "tokens_delivered": 17551, + "latency_ms": 4309.359, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 15143, + "latency_ms": 2943.01, + "indexed_revision_matches": true + }, + { + "provider_id": "code-review-graph", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15992, + "latency_ms": 2755.341, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/codegraph.json b/benchmarks/context-retrieval/results/full-field-got/codegraph.json new file mode 100644 index 00000000..6464b977 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/codegraph.json @@ -0,0 +1,3170 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "codegraph", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "source/core/utils/dns-ip-version.ts", + "source/core/index.ts", + "source/create.ts", + "source/types.ts", + "source/core/utils/weakable-map.ts" + ] + }, + { + "provider_id": "codegraph", + "query": "Fix hanging promise when using cache", + "revision": "633e46e8cb184e6a2670ffae5f8184fcf710f31b", + "returned": [ + "source/as-promise/index.ts", + "source/create.ts", + "source/core/index.ts", + "source/as-promise/types.ts", + "source/types.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "codegraph", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.437, + "full_recall_at_5": 0.1667, + "mean_recall_at_10": 0.5728, + "full_recall_at_10": 0.2963, + "mean_recall_at_20": 0.5775, + "full_recall_at_20": 0.3056, + "mean_recall_at_1000_tokens": 0.2464, + "mean_recall_at_4000_tokens": 0.559, + "mean_recall_at_16000_tokens": 0.5775, + "mean_precision_at_10": 0.1929, + "zero_hit_rate": 0.1481, + "median_tokens_delivered": 3252.5, + "median_latency_ms": 1976.2565, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 76, + "mean_recall_at_5": 0.3272, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.393, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3996, + "full_recall_at_20": 0.0132, + "mean_recall_at_1000_tokens": 0.1572, + "mean_recall_at_4000_tokens": 0.3864, + "mean_recall_at_16000_tokens": 0.3996, + "mean_precision_at_10": 0.1878, + "zero_hit_rate": 0.2105, + "median_tokens_delivered": 3222, + "median_latency_ms": 1995.5615, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 32, + "mean_recall_at_5": 0.6979, + "full_recall_at_5": 0.5625, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.4583, + "mean_recall_at_4000_tokens": 0.9688, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.205, + "zero_hit_rate": 0, + "median_tokens_delivered": 3621.5, + "median_latency_ms": 1929.7025, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3926, + "full_recall_at_5": 0.1111, + "mean_recall_at_10": 0.5654, + "full_recall_at_10": 0.2593, + "mean_recall_at_20": 0.5654, + "full_recall_at_20": 0.2593, + "mean_recall_at_1000_tokens": 0.1765, + "mean_recall_at_4000_tokens": 0.5377, + "mean_recall_at_16000_tokens": 0.5654, + "mean_precision_at_10": 0.2049, + "zero_hit_rate": 0.1296, + "median_tokens_delivered": 3249.5, + "median_latency_ms": 1996.1405, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.4815, + "full_recall_at_5": 0.2222, + "mean_recall_at_10": 0.5802, + "full_recall_at_10": 0.3333, + "mean_recall_at_20": 0.5895, + "full_recall_at_20": 0.3519, + "mean_recall_at_1000_tokens": 0.3164, + "mean_recall_at_4000_tokens": 0.5802, + "mean_recall_at_16000_tokens": 0.5895, + "mean_precision_at_10": 0.1808, + "zero_hit_rate": 0.1667, + "median_tokens_delivered": 3252.5, + "median_latency_ms": 1949.249, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 106, + "indexed-but-returned-nothing": 2 + }, + "resources": { + "baseline_rss_mb": 18891.1, + "peak_rss_mb": 23255, + "delta_rss_mb": 4363.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "codegraph", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1728, + "latency_ms": 2725.393, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2134, + "latency_ms": 2160.141, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4839, + "latency_ms": 1605.088, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3477, + "latency_ms": 1741.697, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2485, + "latency_ms": 2229.228, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0.75, + "precision_at_5": 0.6, + "recall_at_10": 0.75, + "precision_at_10": 0.6, + "recall_at_20": 0.75, + "precision_at_20": 0.6, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 2, + "tokens_delivered": 3570, + "latency_ms": 2906.625, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3943, + "latency_ms": 2807.217, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3108, + "latency_ms": 4146.653, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 4382, + "latency_ms": 2098.747, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3162, + "latency_ms": 1891.542, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3248, + "latency_ms": 2540.654, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2970, + "latency_ms": 2141.858, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.125, + "recall_at_20": 0.3333, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 4472, + "latency_ms": 2250.203, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 3558, + "latency_ms": 2146.137, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 3202, + "latency_ms": 2001.148, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4876, + "latency_ms": 1757.273, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 3711, + "latency_ms": 1945.583, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2777, + "latency_ms": 1952.915, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 4096, + "latency_ms": 1847.025, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2689, + "latency_ms": 2105.015, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2524, + "latency_ms": 2269.223, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3228, + "latency_ms": 2006.436, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2540, + "latency_ms": 2258.05, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 2910, + "latency_ms": 1611.229, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3242, + "latency_ms": 1705.511, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4793, + "latency_ms": 1999.99, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3454, + "latency_ms": 2206.202, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4752, + "latency_ms": 1851.612, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4928, + "latency_ms": 2119.782, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4884, + "latency_ms": 2183.028, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.3333, + "recall_at_10": 0.25, + "precision_at_10": 0.3333, + "recall_at_20": 0.25, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 2148, + "latency_ms": 1750.891, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2445, + "latency_ms": 1756.74, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 4802, + "latency_ms": 1546.354, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.25, + "recall_at_10": 0.3333, + "precision_at_10": 0.25, + "recall_at_20": 0.3333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 2089, + "latency_ms": 2381.342, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4933, + "latency_ms": 1713.966, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 2603, + "latency_ms": 2031.48, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4371, + "latency_ms": 2024.167, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4709, + "latency_ms": 1632.077, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19, + "latency_ms": 1508.407, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1111, + "recall_at_20": 0.3333, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 1439, + "latency_ms": 2309.628, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2999, + "latency_ms": 2041.291, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2507, + "latency_ms": 1900.218, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4482, + "latency_ms": 1928.75, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4919, + "latency_ms": 1600.914, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3842, + "latency_ms": 1920.376, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2554, + "latency_ms": 1891.516, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2286, + "latency_ms": 1580.208, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3257, + "latency_ms": 1632.976, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 3257, + "latency_ms": 4034.32, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2694, + "latency_ms": 1803.982, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2478, + "latency_ms": 1871.576, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts" + ], + "missed": [ + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1506, + "latency_ms": 2280.39, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4553, + "latency_ms": 2131.672, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3620, + "latency_ms": 2053.917, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2308, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4774, + "latency_ms": 1939.029, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1522, + "latency_ms": 1881.692, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3169, + "latency_ms": 1822.641, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 3157, + "latency_ms": 2178.906, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 4761, + "latency_ms": 2029.034, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 3484, + "latency_ms": 1933.053, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4935, + "latency_ms": 1631.285, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 3184, + "latency_ms": 1542.087, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4126, + "latency_ms": 2150.012, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3474, + "latency_ms": 1855.892, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 4150, + "latency_ms": 1618.615, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 2446, + "latency_ms": 1975.496, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3017, + "latency_ms": 3623.392, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2970, + "latency_ms": 2693.624, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4015, + "latency_ms": 1991.133, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 4268, + "latency_ms": 1803.362, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3867, + "latency_ms": 2533.315, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3426, + "latency_ms": 2096.96, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "test/post.ts" + ], + "missed": [ + "source/core/utils/get-body-size.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 3536, + "latency_ms": 2385.014, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.25, + "recall_at_10": 0.3333, + "precision_at_10": 0.25, + "recall_at_20": 0.3333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 4062, + "latency_ms": 2177.405, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2551, + "latency_ms": 1836.846, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.25, + "recall_at_10": 0.3333, + "precision_at_10": 0.25, + "recall_at_20": 0.3333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 3009, + "latency_ms": 2173.771, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4208, + "latency_ms": 1977.017, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2198, + "latency_ms": 2085.287, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2818, + "latency_ms": 1783.759, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16, + "latency_ms": 1792.619, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2893, + "latency_ms": 2184.616, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4895, + "latency_ms": 1764.177, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3542, + "latency_ms": 2556.998, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 3747, + "latency_ms": 2800.268, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1232, + "latency_ms": 2364.584, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2774, + "latency_ms": 2066.363, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 2610, + "latency_ms": 2103.932, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2666, + "latency_ms": 2650.597, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.5, + "recall_at_10": 0.6667, + "precision_at_10": 0.5, + "recall_at_20": 0.6667, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 1121, + "latency_ms": 1618.988, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3351, + "latency_ms": 1915.95, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2741, + "latency_ms": 1738.292, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4262, + "latency_ms": 1877.347, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 2864, + "latency_ms": 2486.299, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3007, + "latency_ms": 2442.718, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 4216, + "latency_ms": 1502.748, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 2776, + "latency_ms": 2014.645, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3623, + "latency_ms": 2169.713, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3983, + "latency_ms": 1770.842, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 2436, + "latency_ms": 1621.114, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 3343, + "latency_ms": 1933.675, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2324, + "latency_ms": 1874.949, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2623, + "latency_ms": 1716.316, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3533, + "latency_ms": 1671.407, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4929, + "latency_ms": 1660.982, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3728, + "latency_ms": 1862.451, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts" + ], + "missed": [ + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.2, + "recall_at_20": 0.2, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 3816, + "latency_ms": 3549.874, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 4626, + "latency_ms": 1636.596, + "indexed_revision_matches": true + }, + { + "provider_id": "codegraph", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2938, + "latency_ms": 1625.796, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/codevetter-structural-context.json b/benchmarks/context-retrieval/results/full-field-got/codevetter-structural-context.json new file mode 100644 index 00000000..763e19e5 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/codevetter-structural-context.json @@ -0,0 +1,3134 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "codevetter-structural-context", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "test/timeout.ts", + "test/cache.ts", + "source/core/index.ts", + "test/arguments.ts", + "source/core/utils/dns-ip-version.ts" + ] + }, + { + "provider_id": "codevetter-structural-context", + "query": "Fix timeout error firing even though it's cancelled (#2399)", + "revision": "f997bcb385d4b35ff581333e6bc9360d5ee10e71", + "returned": [ + "test/stream.ts", + "source/core/errors.ts", + "source/core/index.ts", + "test/error.ts", + "test/timeout.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.5227, + "full_recall_at_5": 0.2778, + "mean_recall_at_10": 0.6938, + "full_recall_at_10": 0.5278, + "mean_recall_at_20": 0.8086, + "full_recall_at_20": 0.6944, + "mean_recall_at_1000_tokens": 0.7941, + "mean_recall_at_4000_tokens": 0.8086, + "mean_recall_at_16000_tokens": 0.8086, + "mean_precision_at_10": 0.1412, + "zero_hit_rate": 0.0833, + "median_tokens_delivered": 927, + "median_latency_ms": 270.9585, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 51, + "mean_recall_at_5": 0.2931, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.3516, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.5948, + "full_recall_at_20": 0.3529, + "mean_recall_at_1000_tokens": 0.5641, + "mean_recall_at_4000_tokens": 0.5948, + "mean_recall_at_16000_tokens": 0.5948, + "mean_precision_at_10": 0.0946, + "zero_hit_rate": 0.1765, + "median_tokens_delivered": 914, + "median_latency_ms": 248.907, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 57, + "mean_recall_at_5": 0.7281, + "full_recall_at_5": 0.5263, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 1, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1828, + "zero_hit_rate": 0, + "median_tokens_delivered": 931, + "median_latency_ms": 274.512, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.513, + "full_recall_at_5": 0.2593, + "mean_recall_at_10": 0.7148, + "full_recall_at_10": 0.5185, + "mean_recall_at_20": 0.8488, + "full_recall_at_20": 0.7222, + "mean_recall_at_1000_tokens": 0.8198, + "mean_recall_at_4000_tokens": 0.8488, + "mean_recall_at_16000_tokens": 0.8488, + "mean_precision_at_10": 0.1431, + "zero_hit_rate": 0.037, + "median_tokens_delivered": 945, + "median_latency_ms": 274.881, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.5324, + "full_recall_at_5": 0.2963, + "mean_recall_at_10": 0.6728, + "full_recall_at_10": 0.537, + "mean_recall_at_20": 0.7685, + "full_recall_at_20": 0.6667, + "mean_recall_at_1000_tokens": 0.7685, + "mean_recall_at_4000_tokens": 0.7685, + "mean_recall_at_16000_tokens": 0.7685, + "mean_precision_at_10": 0.1393, + "zero_hit_rate": 0.1296, + "median_tokens_delivered": 902.5, + "median_latency_ms": 263.0025, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 17664.5, + "peak_rss_mb": 20583.6, + "delta_rss_mb": 2919.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 534724608 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "codevetter-structural-context", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 849, + "latency_ms": 203.499, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1264, + "latency_ms": 248.468, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 740, + "latency_ms": 350.643, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 855, + "latency_ms": 194.944, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 19, + "tokens_delivered": 822, + "latency_ms": 212.42, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1159, + "latency_ms": 279.599, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1518, + "latency_ms": 217.809, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 722, + "latency_ms": 248.467, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 931, + "latency_ms": 325.555, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 665, + "latency_ms": 189.875, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1250, + "latency_ms": 286.67, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 1937, + "latency_ms": 187.368, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 973, + "latency_ms": 165.502, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 891, + "latency_ms": 176.656, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 953, + "latency_ms": 178.139, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 916, + "latency_ms": 465.398, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1000, + "latency_ms": 196.234, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 858, + "latency_ms": 172.445, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 971, + "latency_ms": 210.88, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1387, + "latency_ms": 214.407, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 542, + "latency_ms": 223.837, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 979, + "latency_ms": 168.869, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 1131, + "latency_ms": 345.403, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1874, + "latency_ms": 234.788, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 488, + "latency_ms": 239.973, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1480, + "latency_ms": 210.179, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1052, + "latency_ms": 211.334, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 519, + "latency_ms": 292.436, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1062, + "latency_ms": 243.037, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 727, + "latency_ms": 280.766, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/response.ts" + ], + "missed": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 69, + "latency_ms": 201.992, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 1083, + "latency_ms": 186.92, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 733, + "latency_ms": 180.073, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 870, + "latency_ms": 264.685, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 914, + "latency_ms": 248.907, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [ + "source/types.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 858, + "latency_ms": 241.696, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 797, + "latency_ms": 245.199, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 683, + "latency_ms": 290.847, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [ + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 554, + "latency_ms": 196.954, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 822, + "latency_ms": 326.655, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 321, + "latency_ms": 273.145, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1082, + "latency_ms": 281.542, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 679, + "latency_ms": 264.699, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 432, + "latency_ms": 168.225, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 211, + "latency_ms": 214.419, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 475, + "latency_ms": 198.087, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 785, + "latency_ms": 252.498, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 803, + "latency_ms": 306.532, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1779, + "latency_ms": 269.414, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.4, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 7, + "tokens_delivered": 777, + "latency_ms": 195.385, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1114, + "latency_ms": 251.388, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 790, + "latency_ms": 172.739, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 979, + "latency_ms": 301.247, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 736, + "latency_ms": 274.317, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1396, + "latency_ms": 244.295, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 414, + "latency_ms": 247.182, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 1045, + "latency_ms": 304.77, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 835, + "latency_ms": 205.411, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1125, + "latency_ms": 170.051, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 957, + "latency_ms": 202.05, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 962, + "latency_ms": 216.598, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 1157, + "latency_ms": 222.261, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1196, + "latency_ms": 193.734, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1082, + "latency_ms": 311.985, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1064, + "latency_ms": 314.776, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 745, + "latency_ms": 379.362, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2046, + "latency_ms": 289.503, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 680, + "latency_ms": 276.617, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1418, + "latency_ms": 215.989, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 881, + "latency_ms": 303.561, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 866, + "latency_ms": 219.906, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1087, + "latency_ms": 287.708, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 802, + "latency_ms": 325.537, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1152, + "latency_ms": 226.306, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 937, + "latency_ms": 327.209, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1518, + "latency_ms": 395.335, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 821, + "latency_ms": 280.027, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1428, + "latency_ms": 355.483, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 923, + "latency_ms": 312.728, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 248, + "latency_ms": 345.414, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1171, + "latency_ms": 699.949, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 938, + "latency_ms": 579.733, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 736, + "latency_ms": 369.645, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 876, + "latency_ms": 342.191, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 676, + "latency_ms": 315.526, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1039, + "latency_ms": 311.663, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 1125, + "latency_ms": 294.878, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1202, + "latency_ms": 288.838, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 800, + "latency_ms": 250.925, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 891, + "latency_ms": 274.512, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1021, + "latency_ms": 309.705, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1062, + "latency_ms": 328.058, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1020, + "latency_ms": 272.503, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 56, + "latency_ms": 261.136, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 862, + "latency_ms": 227.275, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1050, + "latency_ms": 367.403, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1033, + "latency_ms": 261.32, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 700, + "latency_ms": 416.941, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 898, + "latency_ms": 276.785, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1410, + "latency_ms": 304.341, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 621, + "latency_ms": 275.46, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1068, + "latency_ms": 380.657, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 950, + "latency_ms": 359.76, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 909, + "latency_ms": 327.193, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 753, + "latency_ms": 402.131, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts" + ], + "missed": [ + "source/types.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 5, + "tokens_delivered": 1558, + "latency_ms": 390.961, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 959, + "latency_ms": 308.438, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 1084, + "latency_ms": 314.553, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/filename-match.json b/benchmarks/context-retrieval/results/full-field-got/filename-match.json new file mode 100644 index 00000000..ca96fe44 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/filename-match.json @@ -0,0 +1,3145 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "filename-match", + "query": "Fix encoding with json `responseType` (#1996)", + "revision": "dbbd31777b3299ceb0db99798453e01ab8e597b1", + "returned": [ + "test/response-parse.ts", + "source/core/response.ts" + ] + }, + { + "provider_id": "filename-match", + "query": "Fixed deprecationWarning on https options (#1391)", + "revision": "a748343363ecb0347897264fb49df4a8f4793997", + "returned": [ + "source/utils/deprecation-warning.ts", + "test/https.ts", + "test/url-to-options.ts", + "source/core/utils/options-to-url.ts", + "source/core/utils/url-to-options.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "filename-match", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.2225, + "full_recall_at_5": 0.0556, + "mean_recall_at_10": 0.2488, + "full_recall_at_10": 0.0833, + "mean_recall_at_20": 0.2858, + "full_recall_at_20": 0.1204, + "mean_recall_at_1000_tokens": 0.0201, + "mean_recall_at_4000_tokens": 0.1315, + "mean_recall_at_16000_tokens": 0.2441, + "mean_precision_at_10": 0.1578, + "zero_hit_rate": 0.537, + "median_tokens_delivered": 3074.5, + "median_latency_ms": 16.2855, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 99, + "mean_recall_at_5": 0.1721, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1805, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2209, + "full_recall_at_20": 0.0404, + "mean_recall_at_1000_tokens": 0.0168, + "mean_recall_at_4000_tokens": 0.098, + "mean_recall_at_16000_tokens": 0.1754, + "mean_precision_at_10": 0.1555, + "zero_hit_rate": 0.5859, + "median_tokens_delivered": 2956, + "median_latency_ms": 16.165, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 9, + "mean_recall_at_5": 0.7778, + "full_recall_at_5": 0.6667, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.0556, + "mean_recall_at_4000_tokens": 0.5, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1826, + "zero_hit_rate": 0, + "median_tokens_delivered": 23892, + "median_latency_ms": 18.57, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.4451, + "full_recall_at_5": 0.1111, + "mean_recall_at_10": 0.4975, + "full_recall_at_10": 0.1667, + "mean_recall_at_20": 0.5716, + "full_recall_at_20": 0.2407, + "mean_recall_at_1000_tokens": 0.0401, + "mean_recall_at_4000_tokens": 0.263, + "mean_recall_at_16000_tokens": 0.4883, + "mean_precision_at_10": 0.3155, + "zero_hit_rate": 0.0741, + "median_tokens_delivered": 7691.5, + "median_latency_ms": 16.754, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 1544, + "median_latency_ms": 15.532, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 88, + "indexed-but-returned-nothing": 20 + }, + "resources": { + "baseline_rss_mb": 16893.7, + "peak_rss_mb": 17224.3, + "delta_rss_mb": 330.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "filename-match", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2269, + "latency_ms": 24.095, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28459, + "latency_ms": 21.716, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1653, + "latency_ms": 16.403, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 7.843, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2917, + "latency_ms": 17.735, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3336, + "latency_ms": 17.267, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44714, + "latency_ms": 33.752, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 24414, + "latency_ms": 25.063, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 312, + "latency_ms": 17.515, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 7.269, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2956, + "latency_ms": 14.924, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 41174, + "latency_ms": 19.161, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2262, + "latency_ms": 20.086, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 37119, + "latency_ms": 18.765, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 5049, + "latency_ms": 18.69, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 9549, + "latency_ms": 22.144, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2605, + "latency_ms": 25.486, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 95, + "latency_ms": 18.426, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 5718, + "latency_ms": 18.269, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 8.822, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 8.056, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4758, + "latency_ms": 17.778, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3, + "latency_ms": 25.365, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 43991, + "latency_ms": 34.216, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 4341, + "latency_ms": 31.021, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 10.133, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1886, + "latency_ms": 17.834, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 11.007, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1831, + "latency_ms": 18.82, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1470, + "latency_ms": 16.929, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 8.129, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3173, + "latency_ms": 18.002, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 33872, + "latency_ms": 18.619, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 8.84, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 9.905, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts", + "source/types.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1111, + "recall_at_20": 0.3333, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 36384, + "latency_ms": 17.345, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3979, + "latency_ms": 16.018, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 6767, + "latency_ms": 15.594, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 8.155, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 7.702, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 23892, + "latency_ms": 15.058, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3871, + "latency_ms": 18.316, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7258, + "latency_ms": 20.191, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 8.836, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 6575, + "latency_ms": 16.868, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3360, + "latency_ms": 15.263, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "test/agent.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1284, + "latency_ms": 16.802, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 10.8, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2222, + "recall_at_20": 0.6667, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 5, + "tokens_delivered": 10804, + "latency_ms": 17.167, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 7.971, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3041, + "latency_ms": 15.859, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1261, + "latency_ms": 16.165, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3678, + "latency_ms": 14.875, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 340, + "latency_ms": 15.472, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/retry.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 8156, + "latency_ms": 16.621, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 192, + "latency_ms": 14.577, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 6552, + "latency_ms": 15.397, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39781, + "latency_ms": 16.706, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1470, + "latency_ms": 15.487, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 371, + "latency_ms": 16.411, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 10.084, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1662, + "latency_ms": 21.674, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3288, + "latency_ms": 17.982, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 7329, + "latency_ms": 18.182, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 12472, + "latency_ms": 18.57, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 11329, + "latency_ms": 16.638, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [ + "test/progress.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 42213, + "latency_ms": 17.057, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 42247, + "latency_ms": 15.179, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2542, + "latency_ms": 17.109, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1653, + "latency_ms": 16.968, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 5361, + "latency_ms": 17.15, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1820, + "latency_ms": 14.327, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2326, + "latency_ms": 15.763, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1667, + "recall_at_20": 0.3333, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 9077, + "latency_ms": 15.847, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2217, + "latency_ms": 15.868, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.25, + "recall_at_20": 0.6667, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 24484, + "latency_ms": 14.038, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2580, + "latency_ms": 15.541, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 8093, + "latency_ms": 15.303, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 25938, + "latency_ms": 19.222, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 11.471, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 18998, + "latency_ms": 19.715, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1653, + "latency_ms": 16.496, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 44561, + "latency_ms": 18.88, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2973, + "latency_ms": 16.006, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1840, + "latency_ms": 15.081, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2991, + "latency_ms": 15.523, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3045, + "latency_ms": 15.147, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 8054, + "latency_ms": 16.504, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 7.837, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1618, + "latency_ms": 17.774, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4022, + "latency_ms": 14.44, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 30717, + "latency_ms": 15.903, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 30332, + "latency_ms": 14.992, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 6.131, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4783, + "latency_ms": 15.338, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5735, + "latency_ms": 14.899, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5290, + "latency_ms": 16.168, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 37166, + "latency_ms": 17.12, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "source/core/index.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 757, + "latency_ms": 16.425, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "test/retry.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 13774, + "latency_ms": 20.687, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 9.031, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 7.648, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2151, + "latency_ms": 17.239, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 5198, + "latency_ms": 16.567, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 5141, + "latency_ms": 15.839, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "test/extend.types.ts" + ], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.3333, + "recall_at_10": 0.2, + "precision_at_10": 0.3333, + "recall_at_20": 0.2, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 4483, + "latency_ms": 14.198, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3104, + "latency_ms": 15.984, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 7286, + "latency_ms": 14.376, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/gitnexus.json b/benchmarks/context-retrieval/results/full-field-got/gitnexus.json new file mode 100644 index 00000000..cfaa9d41 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/gitnexus.json @@ -0,0 +1,3161 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "gitnexus", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "source/index.ts", + "test/timeout.ts" + ] + }, + { + "provider_id": "gitnexus", + "query": "Fix EPIPE errors bypassing retry logic in Promise API", + "revision": "e09a9bd1ca9747f3b53666dc0bca06e1d742899f", + "returned": [ + "source/as-promise/index.ts", + "source/core/response.ts", + "source/core/timed-out.ts", + "source/core/index.ts", + "source/create.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "gitnexus", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.5381, + "full_recall_at_5": 0.2963, + "mean_recall_at_10": 0.648, + "full_recall_at_10": 0.4074, + "mean_recall_at_20": 0.6619, + "full_recall_at_20": 0.4259, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0849, + "mean_recall_at_16000_tokens": 0.5281, + "mean_precision_at_10": 0.2217, + "zero_hit_rate": 0.0926, + "median_tokens_delivered": 16385, + "median_latency_ms": 17978.0495, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 64, + "mean_recall_at_5": 0.3339, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.406, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.4294, + "full_recall_at_20": 0.0313, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0664, + "mean_recall_at_16000_tokens": 0.3026, + "mean_precision_at_10": 0.1979, + "zero_hit_rate": 0.1563, + "median_tokens_delivered": 16385, + "median_latency_ms": 18621.6195, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 44, + "mean_recall_at_5": 0.8352, + "full_recall_at_5": 0.7273, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1117, + "mean_recall_at_16000_tokens": 0.8561, + "mean_precision_at_10": 0.2563, + "zero_hit_rate": 0, + "median_tokens_delivered": 16385, + "median_latency_ms": 17048.2555, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.5778, + "full_recall_at_5": 0.3519, + "mean_recall_at_10": 0.6988, + "full_recall_at_10": 0.4815, + "mean_recall_at_20": 0.6988, + "full_recall_at_20": 0.4815, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0741, + "mean_recall_at_16000_tokens": 0.5685, + "mean_precision_at_10": 0.233, + "zero_hit_rate": 0.0741, + "median_tokens_delivered": 16385, + "median_latency_ms": 17688.445, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.4985, + "full_recall_at_5": 0.2407, + "mean_recall_at_10": 0.5972, + "full_recall_at_10": 0.3333, + "mean_recall_at_20": 0.625, + "full_recall_at_20": 0.3704, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0957, + "mean_recall_at_16000_tokens": 0.4877, + "mean_precision_at_10": 0.2103, + "zero_hit_rate": 0.1111, + "median_tokens_delivered": 16385, + "median_latency_ms": 18705.6465, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 18408.5, + "peak_rss_mb": 43315.9, + "delta_rss_mb": 24907.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "gitnexus", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 16072.971, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12765, + "latency_ms": 18788.512, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 16385, + "latency_ms": 20805.363, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 17715.971, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 17217.353, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/timed-out.ts" + ], + "missed": [ + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2222, + "recall_at_20": 0.5, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 20822.45, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 16191.35, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 18089.653, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 18667.537, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 13611, + "latency_ms": 19084.426, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 21028.118, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15282, + "latency_ms": 18618.474, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/normalize-arguments.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 15662.364, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 14969.852, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 17866.446, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 12986, + "latency_ms": 17621.402, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 14534.458, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 20168.665, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 28467, + "latency_ms": 17612.656, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 17805.477, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 16495.654, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 18758.709, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 16642.864, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 16658.721, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 14825, + "latency_ms": 20531.599, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 16385, + "latency_ms": 31017.907, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 42757.151, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 33559.344, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 15321, + "latency_ms": 37219.562, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 26449.495, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.75, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 21307.198, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 21810.062, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 25725.857, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 22001.348, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 17203.746, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1429, + "recall_at_20": 0.3333, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 16385, + "latency_ms": 21253, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 18793.523, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 18163.289, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1111, + "recall_at_20": 0.25, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 17322.151, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1429, + "recall_at_20": 0.3333, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 23576.493, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 16385, + "latency_ms": 16735.426, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 21661.888, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 26412.386, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 26751.695, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4286, + "recall_at_20": 1, + "precision_at_20": 0.4286, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 27896.661, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3899, + "latency_ms": 20817.729, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 27644.604, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 9810, + "latency_ms": 31891.756, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 28104.002, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1429, + "recall_at_20": 0.2, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 6, + "tokens_delivered": 16385, + "latency_ms": 22861.022, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 23424.151, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts" + ], + "missed": [ + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 43253.765, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 34065.407, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 22333.128, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 22327.744, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 20403.673, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 16385, + "latency_ms": 16132.552, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 18661.795, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 15875.892, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 18743.756, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 19521, + "latency_ms": 16692.442, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 16522.076, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 15333.739, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 15495.78, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 17089.66, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 23957.007, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 18222.103, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 20267.651, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 18458.944, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 16385, + "latency_ms": 17259.443, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 19695.74, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 23144.701, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 19767.383, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 22021.991, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 16851.532, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 32769, + "latency_ms": 21548.835, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 9462, + "latency_ms": 17244.979, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 16385, + "latency_ms": 25195.994, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 16385, + "latency_ms": 17538.714, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 20276.643, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 16789.99, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 14220, + "latency_ms": 14614.289, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 15643.393, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 14548.598, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 13933.131, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 16472.464, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 14307.034, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 17755.488, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 16142.005, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 16385, + "latency_ms": 14310.421, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 23407, + "latency_ms": 15548.097, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 18520.771, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 15605, + "latency_ms": 14100.667, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 15474.285, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 16607.703, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 16385, + "latency_ms": 16058.388, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 7286, + "latency_ms": 13641.323, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 14382, + "latency_ms": 14874.361, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 15481.366, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 15433.291, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 13345, + "latency_ms": 16558.468, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 16385, + "latency_ms": 15693.8, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16385, + "latency_ms": 18624.765, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 9137, + "latency_ms": 16436.015, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 13145, + "latency_ms": 14285.163, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts" + ], + "missed": [ + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.2, + "recall_at_20": 0.2, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 16385, + "latency_ms": 17319.5, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 16385, + "latency_ms": 16878.771, + "indexed_revision_matches": true + }, + { + "provider_id": "gitnexus", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 16385, + "latency_ms": 16914.966, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/gortex.json b/benchmarks/context-retrieval/results/full-field-got/gortex.json new file mode 100644 index 00000000..0ef488ba --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/gortex.json @@ -0,0 +1,3118 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "gortex", + "query": "Fix false `ReadError` on responses without `Content-Length`", + "revision": "b03e6f95ebb5fe3c51512fe0190a20385e2f9ee9", + "returned": [ + "documentation/2-options.md", + "source/core/errors.ts", + "source/core/index.ts", + "source/core/options.ts" + ] + }, + { + "provider_id": "gortex", + "query": "Fix stream validation errors causing unhandled rejections", + "revision": "30b3b795580d7d2a62e85382af5483bcc9a60899", + "returned": [ + "documentation/tips.md", + "source/core/index.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "gortex", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.071, + "full_recall_at_5": 0.0278, + "mean_recall_at_10": 0.071, + "full_recall_at_10": 0.0278, + "mean_recall_at_20": 0.071, + "full_recall_at_20": 0.0278, + "mean_recall_at_1000_tokens": 0.0687, + "mean_recall_at_4000_tokens": 0.071, + "mean_recall_at_16000_tokens": 0.071, + "mean_precision_at_10": 0.0833, + "zero_hit_rate": 0.8704, + "median_tokens_delivered": 43, + "median_latency_ms": 4373.983, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 105, + "mean_recall_at_5": 0.0444, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0444, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0444, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.0421, + "mean_recall_at_4000_tokens": 0.0444, + "mean_recall_at_16000_tokens": 0.0444, + "mean_precision_at_10": 0.0603, + "zero_hit_rate": 0.8952, + "median_tokens_delivered": 43, + "median_latency_ms": 4386.534, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 3, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 1, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.8889, + "zero_hit_rate": 0, + "median_tokens_delivered": 275, + "median_latency_ms": 3554.898, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.071, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.071, + "full_recall_at_10": 0.037, + "mean_recall_at_20": 0.071, + "full_recall_at_20": 0.037, + "mean_recall_at_1000_tokens": 0.071, + "mean_recall_at_4000_tokens": 0.071, + "mean_recall_at_16000_tokens": 0.071, + "mean_precision_at_10": 0.0725, + "zero_hit_rate": 0.8889, + "median_tokens_delivered": 45, + "median_latency_ms": 4835.2715, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.071, + "full_recall_at_5": 0.0185, + "mean_recall_at_10": 0.071, + "full_recall_at_10": 0.0185, + "mean_recall_at_20": 0.071, + "full_recall_at_20": 0.0185, + "mean_recall_at_1000_tokens": 0.0664, + "mean_recall_at_4000_tokens": 0.071, + "mean_recall_at_16000_tokens": 0.071, + "mean_precision_at_10": 0.0941, + "zero_hit_rate": 0.8519, + "median_tokens_delivered": 43, + "median_latency_ms": 4198.1405, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 37, + "indexed-but-returned-nothing": 71 + }, + "resources": { + "baseline_rss_mb": 26094.4, + "peak_rss_mb": 43250.6, + "delta_rss_mb": 17156.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "gortex", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 754, + "latency_ms": 21329.896, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 558, + "latency_ms": 4347.233, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1426, + "latency_ms": 3462.572, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 488, + "latency_ms": 4462.631, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 948, + "latency_ms": 2460.374, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.25, + "recall_at_10": 0.25, + "precision_at_10": 0.25, + "recall_at_20": 0.25, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 1474, + "latency_ms": 4361.432, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 714, + "latency_ms": 5190.56, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 773, + "latency_ms": 2726.641, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 446, + "latency_ms": 6711.025, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 950, + "latency_ms": 4228.909, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 569, + "latency_ms": 5121.44, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 112, + "latency_ms": 4582.273, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 623, + "latency_ms": 3649.343, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32, + "latency_ms": 2971.487, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 403, + "latency_ms": 26280.018, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 929, + "latency_ms": 2579.031, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 358, + "latency_ms": 2107.768, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 251, + "latency_ms": 2040.399, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 492, + "latency_ms": 8282.083, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40, + "latency_ms": 2996.849, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38, + "latency_ms": 3228.018, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 275, + "latency_ms": 3554.898, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 276, + "latency_ms": 3676.586, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 221, + "latency_ms": 2851.393, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 373, + "latency_ms": 2484.965, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 264, + "latency_ms": 3356.527, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 239, + "latency_ms": 6365.147, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43, + "latency_ms": 3796.185, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40, + "latency_ms": 3224.577, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37, + "latency_ms": 4131.301, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36, + "latency_ms": 2145.958, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 345, + "latency_ms": 2086.015, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35, + "latency_ms": 1376.404, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42, + "latency_ms": 2536.941, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37, + "latency_ms": 2219.549, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 250, + "latency_ms": 9706.77, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 3545.326, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42, + "latency_ms": 2540.113, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38, + "latency_ms": 1976.443, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45, + "latency_ms": 4473.655, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 234, + "latency_ms": 2961.947, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44, + "latency_ms": 2592.305, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35, + "latency_ms": 4167.372, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36, + "latency_ms": 2272.261, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 266, + "latency_ms": 1988.184, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 248, + "latency_ms": 2212.6, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40, + "latency_ms": 2828.913, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43, + "latency_ms": 2837.462, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41, + "latency_ms": 2266.833, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37, + "latency_ms": 88367.755, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41, + "latency_ms": 20764.403, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.5, + "recall_at_10": 0.25, + "precision_at_10": 0.5, + "recall_at_20": 0.25, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 594, + "latency_ms": 3075.008, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37, + "latency_ms": 3853.924, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38, + "latency_ms": 2129.189, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43, + "latency_ms": 2150.593, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41, + "latency_ms": 88619.496, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37, + "latency_ms": 2426.837, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34, + "latency_ms": 3533.481, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37, + "latency_ms": 5274.084, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41, + "latency_ms": 4969.297, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43, + "latency_ms": 4551.459, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 244, + "latency_ms": 3894.833, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 256, + "latency_ms": 3716.459, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40, + "latency_ms": 7760.615, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39, + "latency_ms": 4918.208, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 274, + "latency_ms": 5428.058, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35, + "latency_ms": 5608.919, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36, + "latency_ms": 3211.862, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 231, + "latency_ms": 4750.271, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40, + "latency_ms": 4099.83, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40, + "latency_ms": 8570.912, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41, + "latency_ms": 5325.495, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38, + "latency_ms": 3776.03, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 180, + "latency_ms": 7607.155, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36, + "latency_ms": 3805.96, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.25, + "recall_at_10": 0.3333, + "precision_at_10": 0.25, + "recall_at_20": 0.3333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 1130, + "latency_ms": 38130.431, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36, + "latency_ms": 7408.492, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37, + "latency_ms": 13277.13, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 392, + "latency_ms": 17650.922, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 212, + "latency_ms": 8700.988, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39, + "latency_ms": 5844.077, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43, + "latency_ms": 9493.83, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33, + "latency_ms": 4043.82, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41, + "latency_ms": 17094.504, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38, + "latency_ms": 16326.428, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38, + "latency_ms": 10299.994, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46, + "latency_ms": 20674.485, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45, + "latency_ms": 24165.768, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39, + "latency_ms": 11707.39, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43, + "latency_ms": 7521.86, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45, + "latency_ms": 3128.566, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 4170.444, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45, + "latency_ms": 11498.231, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33, + "latency_ms": 7763.384, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 251, + "latency_ms": 4246.481, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 50, + "latency_ms": 5078.257, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39, + "latency_ms": 6871.108, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35, + "latency_ms": 4752.335, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45, + "latency_ms": 15301.729, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 10249.413, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36, + "latency_ms": 6307.289, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41, + "latency_ms": 4386.534, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39, + "latency_ms": 4511.415, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35, + "latency_ms": 10537.805, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38, + "latency_ms": 5893.966, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 8787.199, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47, + "latency_ms": 6784.291, + "indexed_revision_matches": true + }, + { + "provider_id": "gortex", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 186, + "latency_ms": 8886.999, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/graft.json b/benchmarks/context-retrieval/results/full-field-got/graft.json new file mode 100644 index 00000000..c0a15a48 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/graft.json @@ -0,0 +1,3179 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "graft", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "source/core/utils/dns-ip-version.ts", + "source/core/index.ts", + "source/index.ts" + ] + }, + { + "provider_id": "graft", + "query": "Fix for sending files with size `0` on `stat` (#1488)", + "revision": "6aa86f2494194907f6c6c8b4774dfa1f69df6876", + "returned": [ + "source/core/utils/get-body-size.ts", + "source/core/index.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "graft", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.4594, + "full_recall_at_5": 0.1296, + "mean_recall_at_10": 0.4687, + "full_recall_at_10": 0.1389, + "mean_recall_at_20": 0.4687, + "full_recall_at_20": 0.1389, + "mean_recall_at_1000_tokens": 0.461, + "mean_recall_at_4000_tokens": 0.4687, + "mean_recall_at_16000_tokens": 0.4687, + "mean_precision_at_10": 0.2832, + "zero_hit_rate": 0.1759, + "median_tokens_delivered": 267, + "median_latency_ms": 2564.4315, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 93, + "mean_recall_at_5": 0.383, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.383, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.383, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.374, + "mean_recall_at_4000_tokens": 0.383, + "mean_recall_at_16000_tokens": 0.383, + "mean_precision_at_10": 0.2803, + "zero_hit_rate": 0.2043, + "median_tokens_delivered": 272, + "median_latency_ms": 2553.968, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 15, + "mean_recall_at_5": 0.9333, + "full_recall_at_5": 0.9333, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 1, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.3011, + "zero_hit_rate": 0, + "median_tokens_delivered": 257, + "median_latency_ms": 2582.798, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.4574, + "full_recall_at_5": 0.0926, + "mean_recall_at_10": 0.4759, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.4759, + "full_recall_at_20": 0.1111, + "mean_recall_at_1000_tokens": 0.4698, + "mean_recall_at_4000_tokens": 0.4759, + "mean_recall_at_16000_tokens": 0.4759, + "mean_precision_at_10": 0.2741, + "zero_hit_rate": 0.1481, + "median_tokens_delivered": 268.5, + "median_latency_ms": 2605.409, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.4614, + "full_recall_at_5": 0.1667, + "mean_recall_at_10": 0.4614, + "full_recall_at_10": 0.1667, + "mean_recall_at_20": 0.4614, + "full_recall_at_20": 0.1667, + "mean_recall_at_1000_tokens": 0.4522, + "mean_recall_at_4000_tokens": 0.4614, + "mean_recall_at_16000_tokens": 0.4614, + "mean_precision_at_10": 0.2923, + "zero_hit_rate": 0.2037, + "median_tokens_delivered": 263, + "median_latency_ms": 2485.713, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 22928.6, + "peak_rss_mb": 39459.7, + "delta_rss_mb": 16531.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "graft", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 247, + "latency_ms": 2247.416, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 255, + "latency_ms": 2262.257, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 273, + "latency_ms": 2179.065, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 212, + "latency_ms": 2876.217, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 274, + "latency_ms": 3292.165, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.3333, + "recall_at_10": 0.25, + "precision_at_10": 0.3333, + "recall_at_20": 0.25, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 254, + "latency_ms": 3169.007, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 276, + "latency_ms": 2137.914, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 269, + "latency_ms": 1683.989, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 283, + "latency_ms": 2897.793, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 323, + "latency_ms": 2093.7, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 672, + "latency_ms": 4728.874, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 291, + "latency_ms": 1874.504, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 239, + "latency_ms": 1552.736, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 256, + "latency_ms": 2427.858, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 249, + "latency_ms": 2637.394, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 332, + "latency_ms": 2413.246, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 290, + "latency_ms": 1932.848, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 286, + "latency_ms": 1807.114, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 258, + "latency_ms": 2132.961, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 201, + "latency_ms": 1977.745, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 914, + "latency_ms": 1980.828, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 264, + "latency_ms": 1609.864, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 302, + "latency_ms": 2327.826, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 333, + "latency_ms": 2695.589, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 251, + "latency_ms": 2332.063, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 238, + "latency_ms": 2417.458, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 658, + "latency_ms": 3182.574, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 259, + "latency_ms": 3427.046, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 283, + "latency_ms": 3798.689, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 230, + "latency_ms": 2136.003, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/response.ts" + ], + "missed": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 289, + "latency_ms": 2165.617, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 240, + "latency_ms": 2296.353, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 255, + "latency_ms": 1957.57, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 281, + "latency_ms": 2683.501, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 244, + "latency_ms": 2574.895, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.5, + "recall_at_10": 0.6667, + "precision_at_10": 0.5, + "recall_at_20": 0.6667, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 264, + "latency_ms": 3029.517, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 266, + "latency_ms": 2651.857, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 268, + "latency_ms": 2136.953, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.3333, + "recall_at_10": 0.25, + "precision_at_10": 0.3333, + "recall_at_20": 0.25, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 262, + "latency_ms": 1449.175, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 327, + "latency_ms": 4052.347, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 281, + "latency_ms": 2617.22, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 311, + "latency_ms": 2252.815, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 280, + "latency_ms": 3098.408, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 222, + "latency_ms": 2335.878, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/types.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 307, + "latency_ms": 1757.882, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45, + "latency_ms": 2152.844, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 259, + "latency_ms": 2992.846, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 1, + "recall_at_10": 0.6667, + "precision_at_10": 1, + "recall_at_20": 0.6667, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 357, + "latency_ms": 2553.968, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 240, + "latency_ms": 2593.598, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 246, + "latency_ms": 2324.153, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 274, + "latency_ms": 2080.124, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.25, + "recall_at_10": 0.25, + "precision_at_10": 0.25, + "recall_at_20": 0.25, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 230, + "latency_ms": 5621.869, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 249, + "latency_ms": 4737.088, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 258, + "latency_ms": 1799.021, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts" + ], + "missed": [ + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.25, + "recall_at_10": 0.3333, + "precision_at_10": 0.25, + "recall_at_20": 0.3333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 259, + "latency_ms": 2127.264, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 251, + "latency_ms": 2009.232, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 241, + "latency_ms": 1496.119, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 288, + "latency_ms": 1987.514, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 258, + "latency_ms": 1803.157, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 251, + "latency_ms": 1940.88, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 256, + "latency_ms": 2375.454, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.25, + "recall_at_10": 0.3333, + "precision_at_10": 0.25, + "recall_at_20": 0.3333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 258, + "latency_ms": 2313.534, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 303, + "latency_ms": 1869.919, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 248, + "latency_ms": 2259.005, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 262, + "latency_ms": 2071.288, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.5, + "recall_at_10": 0.6667, + "precision_at_10": 0.5, + "recall_at_20": 0.6667, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 2127, + "latency_ms": 3069.068, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 285, + "latency_ms": 2088.69, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 228, + "latency_ms": 2232.158, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 185, + "latency_ms": 1715.636, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 272, + "latency_ms": 2914.258, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 282, + "latency_ms": 2483.669, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 708, + "latency_ms": 2310.265, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/post.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 279, + "latency_ms": 2944.495, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.25, + "recall_at_10": 0.3333, + "precision_at_10": 0.25, + "recall_at_20": 0.3333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 266, + "latency_ms": 2363.746, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 238, + "latency_ms": 2134.551, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 472, + "latency_ms": 6262.192, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 259, + "latency_ms": 2582.798, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 432, + "latency_ms": 4478.5, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 820, + "latency_ms": 9282.97, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 257, + "latency_ms": 6078.743, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2195, + "latency_ms": 4995.742, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 978, + "latency_ms": 2741.897, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 223, + "latency_ms": 3642.517, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 237, + "latency_ms": 2967.768, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 253, + "latency_ms": 2282.618, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 314, + "latency_ms": 2677.4, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 307, + "latency_ms": 2265.514, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 266, + "latency_ms": 12253.249, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 255, + "latency_ms": 10932.802, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 308, + "latency_ms": 10357.837, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 280, + "latency_ms": 7898.329, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 333, + "latency_ms": 4674.309, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 284, + "latency_ms": 5617.732, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 700, + "latency_ms": 4557.012, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 240, + "latency_ms": 4560.6, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 313, + "latency_ms": 5074.362, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 239, + "latency_ms": 5858.041, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 254, + "latency_ms": 7083.451, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.5, + "recall_at_10": 0.6667, + "precision_at_10": 0.5, + "recall_at_20": 0.6667, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 239, + "latency_ms": 7556.246, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 676, + "latency_ms": 4505.731, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 237, + "latency_ms": 4449.779, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 283, + "latency_ms": 2838.26, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 262, + "latency_ms": 4306.682, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 270, + "latency_ms": 4354.895, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 277, + "latency_ms": 2547.341, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts" + ], + "missed": [ + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1667, + "recall_at_20": 0.2, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 818, + "latency_ms": 4260.259, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2059, + "latency_ms": 3609.704, + "indexed_revision_matches": true + }, + { + "provider_id": "graft", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 282, + "latency_ms": 9053.428, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/graphify.json b/benchmarks/context-retrieval/results/full-field-got/graphify.json new file mode 100644 index 00000000..aad05add --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/graphify.json @@ -0,0 +1,3143 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "graphify", + "query": "Fix encoding with json `responseType` (#1996)", + "revision": "dbbd31777b3299ceb0db99798453e01ab8e597b1", + "returned": [ + "source/core/options.ts", + "documentation/2-options.md", + "source/core/response.ts", + "source/core/index.ts", + "source/index.ts" + ] + }, + { + "provider_id": "graphify", + "query": "Fix unhandled `The server aborted pending request` rejection", + "revision": "04f3ea4b7995e1609555dd13f00641b8f470de4c", + "returned": [ + "readme.md", + "source/as-promise/create-rejection.ts", + "benchmark/server.ts", + "source/create.ts", + "source/as-promise/types.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.284, + "full_recall_at_5": 0.1204, + "mean_recall_at_10": 0.3957, + "full_recall_at_10": 0.213, + "mean_recall_at_20": 0.4577, + "full_recall_at_20": 0.2963, + "mean_recall_at_1000_tokens": 0.2887, + "mean_recall_at_4000_tokens": 0.4346, + "mean_recall_at_16000_tokens": 0.4577, + "mean_precision_at_10": 0.1039, + "zero_hit_rate": 0.3981, + "median_tokens_delivered": 1677.5, + "median_latency_ms": 2435.545, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 85, + "mean_recall_at_5": 0.1745, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2322, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.311, + "full_recall_at_20": 0.1059, + "mean_recall_at_1000_tokens": 0.1433, + "mean_recall_at_4000_tokens": 0.2875, + "mean_recall_at_16000_tokens": 0.311, + "mean_precision_at_10": 0.0831, + "zero_hit_rate": 0.5059, + "median_tokens_delivered": 1681, + "median_latency_ms": 2363.334, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 23, + "mean_recall_at_5": 0.6884, + "full_recall_at_5": 0.5652, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.8261, + "mean_recall_at_4000_tokens": 0.9783, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1806, + "zero_hit_rate": 0, + "median_tokens_delivered": 1669, + "median_latency_ms": 2620.509, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.2253, + "full_recall_at_5": 0.0741, + "mean_recall_at_10": 0.3407, + "full_recall_at_10": 0.1481, + "mean_recall_at_20": 0.4123, + "full_recall_at_20": 0.2593, + "mean_recall_at_1000_tokens": 0.292, + "mean_recall_at_4000_tokens": 0.4123, + "mean_recall_at_16000_tokens": 0.4123, + "mean_precision_at_10": 0.0852, + "zero_hit_rate": 0.4444, + "median_tokens_delivered": 1676, + "median_latency_ms": 2333.08, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3426, + "full_recall_at_5": 0.1667, + "mean_recall_at_10": 0.4506, + "full_recall_at_10": 0.2778, + "mean_recall_at_20": 0.5031, + "full_recall_at_20": 0.3333, + "mean_recall_at_1000_tokens": 0.2855, + "mean_recall_at_4000_tokens": 0.4568, + "mean_recall_at_16000_tokens": 0.5031, + "mean_precision_at_10": 0.1225, + "zero_hit_rate": 0.3519, + "median_tokens_delivered": 1679.5, + "median_latency_ms": 2557.876, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 17668.2, + "peak_rss_mb": 23201.4, + "delta_rss_mb": 5533.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 84590592 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "graphify", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 15, + "tokens_delivered": 6858, + "latency_ms": 1580.932, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1657, + "latency_ms": 1630.571, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3335, + "latency_ms": 1728.703, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2594, + "latency_ms": 1506.808, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 5791, + "latency_ms": 1989.832, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 2, + "tokens_delivered": 1677, + "latency_ms": 2376.549, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4439, + "latency_ms": 2212.264, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1657, + "latency_ms": 2141.156, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 8203, + "latency_ms": 2549.58, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2927, + "latency_ms": 2132.811, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 6443, + "latency_ms": 3797.403, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2733, + "latency_ms": 2018.779, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 8, + "tokens_delivered": 6565, + "latency_ms": 1781.82, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1668, + "latency_ms": 2106.436, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1696, + "latency_ms": 2279.379, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1687, + "latency_ms": 2306.767, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 166, + "latency_ms": 2241.199, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 160, + "latency_ms": 2752.634, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1662, + "latency_ms": 2589.32, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 492, + "latency_ms": 2522.893, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3555, + "latency_ms": 2760.777, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3960, + "latency_ms": 2631.482, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2833, + "latency_ms": 3165.95, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3776, + "latency_ms": 2717.106, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1674, + "latency_ms": 2877.373, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1665, + "latency_ms": 1796.521, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1663, + "latency_ms": 2363.334, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1677, + "latency_ms": 2790.441, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1663, + "latency_ms": 2365.486, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1660, + "latency_ms": 2394.647, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 4937, + "latency_ms": 2470.177, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 3653, + "latency_ms": 1991.636, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2247, + "latency_ms": 2063.193, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 1668, + "latency_ms": 3865.986, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1681, + "latency_ms": 3422.728, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 1678, + "latency_ms": 4820.222, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1673, + "latency_ms": 2358.804, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1664, + "latency_ms": 2620.509, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 522, + "latency_ms": 2173.036, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 5830, + "latency_ms": 4129.219, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1669, + "latency_ms": 2303.253, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1715, + "latency_ms": 2238.171, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/utils/timer.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 12299, + "latency_ms": 3361.488, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1450, + "latency_ms": 1777.479, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3436, + "latency_ms": 2338.897, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 894, + "latency_ms": 2580.544, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3247, + "latency_ms": 2269.314, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1662, + "latency_ms": 3041.197, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 12, + "tokens_delivered": 1681, + "latency_ms": 2235.943, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4373, + "latency_ms": 2188.864, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1680, + "latency_ms": 2891.416, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 6, + "tokens_delivered": 7242, + "latency_ms": 2137.587, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1143, + "latency_ms": 3638.482, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1669, + "latency_ms": 1792.441, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1662, + "latency_ms": 2108.979, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5707, + "latency_ms": 2432.868, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1415, + "latency_ms": 1995.361, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2113, + "latency_ms": 2111.024, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1344, + "latency_ms": 2123.548, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1667, + "latency_ms": 2112.151, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 155, + "latency_ms": 1832.711, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 2407, + "latency_ms": 2267.911, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3863, + "latency_ms": 2186.971, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3437, + "latency_ms": 2443.543, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 1666, + "latency_ms": 1847.563, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3007, + "latency_ms": 3226.015, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2755, + "latency_ms": 2307.25, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2834, + "latency_ms": 1821.476, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 5412, + "latency_ms": 2318.893, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1674, + "latency_ms": 4271.439, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1654, + "latency_ms": 2255.636, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1676, + "latency_ms": 2964.32, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 5149, + "latency_ms": 2663.578, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1673, + "latency_ms": 2651.977, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1315, + "latency_ms": 2316.014, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1676, + "latency_ms": 3092.955, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1659, + "latency_ms": 2834.241, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1667, + "latency_ms": 3111.207, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1657, + "latency_ms": 2671.168, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2650, + "latency_ms": 2734.94, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3589, + "latency_ms": 2491.982, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1669, + "latency_ms": 2566.172, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1668, + "latency_ms": 5150.352, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 1661, + "latency_ms": 2305.236, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4701, + "latency_ms": 2294.748, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 231, + "latency_ms": 2642.268, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1664, + "latency_ms": 2438.222, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1669, + "latency_ms": 2747.162, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 4189, + "latency_ms": 2831.465, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1680, + "latency_ms": 1860.891, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1668, + "latency_ms": 2175.876, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1691, + "latency_ms": 2165.34, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1676, + "latency_ms": 2440.438, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2832, + "latency_ms": 2667.457, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1312, + "latency_ms": 2327.263, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 1679, + "latency_ms": 2928.148, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1665, + "latency_ms": 3640.625, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2420, + "latency_ms": 2739.91, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 1678, + "latency_ms": 2571.667, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 222, + "latency_ms": 3212.419, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3593, + "latency_ms": 2512.754, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2724, + "latency_ms": 2931.281, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1669, + "latency_ms": 2782.657, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4150, + "latency_ms": 3212.131, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 161, + "latency_ms": 2980.062, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "missed": [ + "test/create.ts", + "test/extend.types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 8, + "tokens_delivered": 1657, + "latency_ms": 2693.995, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1248, + "latency_ms": 3097.667, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 1683, + "latency_ms": 1602.163, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/jcodemunch.json b/benchmarks/context-retrieval/results/full-field-got/jcodemunch.json new file mode 100644 index 00000000..96990e80 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/jcodemunch.json @@ -0,0 +1,3156 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "jcodemunch", + "query": "Fix assertions when cloning raw options", + "revision": "0fb6ec60d299fd9b48966608a4c3f201746d821c", + "returned": [ + "source/core/options.ts", + "source/core/index.ts", + "test/helpers/with-server.ts" + ] + }, + { + "provider_id": "jcodemunch", + "query": "Fix unhandled `The server aborted pending request` rejection", + "revision": "04f3ea4b7995e1609555dd13f00641b8f470de4c", + "returned": [ + "test/cancel.ts", + "source/core/index.ts", + "test/helpers/with-server.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "jcodemunch", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.3596, + "full_recall_at_5": 0.1389, + "mean_recall_at_10": 0.3596, + "full_recall_at_10": 0.1389, + "mean_recall_at_20": 0.3596, + "full_recall_at_20": 0.1389, + "mean_recall_at_1000_tokens": 0.2461, + "mean_recall_at_4000_tokens": 0.3519, + "mean_recall_at_16000_tokens": 0.3596, + "mean_precision_at_10": 0.3031, + "zero_hit_rate": 0.4074, + "median_tokens_delivered": 1164.5, + "median_latency_ms": 927.2845, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 11 + }, + "baseline_missed": { + "cases": 93, + "mean_recall_at_5": 0.2563, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2563, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2563, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.1461, + "mean_recall_at_4000_tokens": 0.2473, + "mean_recall_at_16000_tokens": 0.2563, + "mean_precision_at_10": 0.2749, + "zero_hit_rate": 0.4731, + "median_tokens_delivered": 1359, + "median_latency_ms": 948.387, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 11 + }, + "baseline_solved": { + "cases": 15, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.8667, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.4778, + "zero_hit_rate": 0, + "median_tokens_delivered": 551, + "median_latency_ms": 897.473, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3611, + "full_recall_at_5": 0.1296, + "mean_recall_at_10": 0.3611, + "full_recall_at_10": 0.1296, + "mean_recall_at_20": 0.3611, + "full_recall_at_20": 0.1296, + "mean_recall_at_1000_tokens": 0.2161, + "mean_recall_at_4000_tokens": 0.3457, + "mean_recall_at_16000_tokens": 0.3611, + "mean_precision_at_10": 0.3333, + "zero_hit_rate": 0.4074, + "median_tokens_delivered": 1072.5, + "median_latency_ms": 980.9465, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 4 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.358, + "full_recall_at_5": 0.1481, + "mean_recall_at_10": 0.358, + "full_recall_at_10": 0.1481, + "mean_recall_at_20": 0.358, + "full_recall_at_20": 0.1481, + "mean_recall_at_1000_tokens": 0.2762, + "mean_recall_at_4000_tokens": 0.358, + "mean_recall_at_16000_tokens": 0.358, + "mean_precision_at_10": 0.2728, + "zero_hit_rate": 0.4074, + "median_tokens_delivered": 1232.5, + "median_latency_ms": 902.1245, + "payload_kind": [ + "mcp-tool-result" + ], + "unavailable": 7 + } + }, + "outcomes": { + "answered": 97, + "indexed-but-returned-nothing": 11 + }, + "resources": { + "baseline_rss_mb": 17670.5, + "peak_rss_mb": 25003.7, + "delta_rss_mb": 7333.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "jcodemunch", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 667, + "latency_ms": 499.082, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 554, + "latency_ms": 792.769, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1389, + "latency_ms": 531.722, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 446, + "latency_ms": 566.253, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 730, + "latency_ms": 518.356, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3651, + "latency_ms": 1028.642, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1737, + "latency_ms": 757.906, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1629, + "latency_ms": 764.79, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1273, + "latency_ms": 762.678, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 459, + "latency_ms": 908.769, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1192, + "latency_ms": 1260.934, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1110, + "latency_ms": 707.989, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 521, + "latency_ms": 893.298, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 925, + "latency_ms": 856.148, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2120, + "latency_ms": 885.078, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "test/retry.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2427, + "latency_ms": 736.182, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1912, + "latency_ms": 728.113, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 471, + "latency_ms": 924.23, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 704, + "latency_ms": 804.657, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6068, + "latency_ms": 849.811, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 458, + "latency_ms": 814.6, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2796, + "latency_ms": 905.612, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2569, + "latency_ms": 878.318, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 969, + "latency_ms": 1341.516, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1359, + "latency_ms": 1171.396, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 589, + "latency_ms": 2325.096, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1551, + "latency_ms": 674.599, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1170, + "latency_ms": 1291.877, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3648, + "latency_ms": 1055.51, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1922, + "latency_ms": 564.13, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6667, + "recall_at_10": 0.5, + "precision_at_10": 0.6667, + "recall_at_20": 0.5, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1878, + "latency_ms": 1311.548, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3996, + "latency_ms": 1053.942, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 468, + "latency_ms": 848.658, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 2785, + "latency_ms": 1252.702, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2184, + "latency_ms": 968.653, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 2212, + "latency_ms": 1069.571, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5024, + "latency_ms": 668.53, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3092, + "latency_ms": 1040.347, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.3333, + "recall_at_10": 0.25, + "precision_at_10": 0.3333, + "recall_at_20": 0.25, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 601, + "latency_ms": 668.886, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 484, + "latency_ms": 961.379, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2188, + "latency_ms": 1012.332, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3574, + "latency_ms": 835.627, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 459, + "latency_ms": 962.408, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 566, + "latency_ms": 1130.72, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 2095, + "latency_ms": 872.532, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2872, + "latency_ms": 902.334, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2209, + "latency_ms": 977.277, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5802, + "latency_ms": 930.339, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 3715, + "latency_ms": 1659.171, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 443, + "latency_ms": 995.831, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2219, + "latency_ms": 901.915, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.3333, + "recall_at_10": 0.25, + "precision_at_10": 0.3333, + "recall_at_20": 0.25, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 1118, + "latency_ms": 715.994, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 660, + "latency_ms": 1632.523, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 551, + "latency_ms": 755.707, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.5, + "recall_at_10": 0.6667, + "precision_at_10": 0.5, + "recall_at_20": 0.6667, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 540, + "latency_ms": 1065.423, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 944, + "latency_ms": 797.362, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 520, + "latency_ms": 1173.957, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 420, + "latency_ms": 745.128, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1925, + "latency_ms": 974.531, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 530, + "latency_ms": 1110.059, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4155, + "latency_ms": 875.542, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.5, + "recall_at_10": 0.3333, + "precision_at_10": 0.5, + "recall_at_20": 0.3333, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 3597, + "latency_ms": 794.339, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 599, + "latency_ms": 1066.005, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 635, + "latency_ms": 1062.665, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1875, + "latency_ms": 984.616, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 4086, + "latency_ms": 948.082, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 513, + "latency_ms": 677.793, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 437, + "latency_ms": 1209.065, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 424, + "latency_ms": 1132.496, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 534, + "latency_ms": 1699.754, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 561, + "latency_ms": 897.473, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1018, + "latency_ms": 710.703, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2501, + "latency_ms": 1317.67, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 688, + "latency_ms": 1064.49, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 735, + "latency_ms": 1072.769, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6383, + "latency_ms": 1291.667, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 729, + "latency_ms": 1157.169, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 496, + "latency_ms": 949.47, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1114, + "latency_ms": 1067.672, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105, + "latency_ms": 572.804, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4493, + "latency_ms": 1070.717, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5126, + "latency_ms": 948.387, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 421, + "latency_ms": 1102.505, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2696, + "latency_ms": 703.758, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1816, + "latency_ms": 862.897, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 858, + "latency_ms": 770.42, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3126, + "latency_ms": 1173.551, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 489, + "latency_ms": 799.386, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 459, + "latency_ms": 1281.271, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2005, + "latency_ms": 970.821, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2722, + "latency_ms": 700.812, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3810, + "latency_ms": 685.064, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6923, + "latency_ms": 795.946, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2676, + "latency_ms": 854.798, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2179, + "latency_ms": 721.327, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5703, + "latency_ms": 1158.027, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 550, + "latency_ms": 815.169, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 448, + "latency_ms": 1002.856, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 402, + "latency_ms": 969.732, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1035, + "latency_ms": 1818.756, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 504, + "latency_ms": 813.88, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 453, + "latency_ms": 1035.813, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1159, + "latency_ms": 883.304, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 541, + "latency_ms": 1270.863, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 623, + "latency_ms": 755.07, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7435, + "latency_ms": 1093.344, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2430, + "latency_ms": 665.737, + "indexed_revision_matches": true + }, + { + "provider_id": "jcodemunch", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2430, + "latency_ms": 1134.202, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/keyword-search.json b/benchmarks/context-retrieval/results/full-field-got/keyword-search.json new file mode 100644 index 00000000..4622c24f --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/keyword-search.json @@ -0,0 +1,3133 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "keyword-search", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "test/timeout.ts", + "package.json", + "source/create.ts", + "source/index.ts", + "source/types.ts" + ] + }, + { + "provider_id": "keyword-search", + "query": "Fix a regression where body was sent after redirect", + "revision": "25ae938e441ee1fb35bcfa76582d1075cc5c9eef", + "returned": [ + "test/agent.ts", + "test/cancel.ts", + "test/stream.ts", + "source/core/index.ts", + "source/create.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "keyword-search", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.4119, + "full_recall_at_5": 0.1944, + "mean_recall_at_10": 0.5809, + "full_recall_at_10": 0.3611, + "mean_recall_at_20": 0.81, + "full_recall_at_20": 0.6944, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1065, + "mean_recall_at_16000_tokens": 0.391, + "mean_precision_at_10": 0.1208, + "zero_hit_rate": 0.0926, + "median_tokens_delivered": 62238.5, + "median_latency_ms": 49.2595, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 69, + "mean_recall_at_5": 0.2304, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.344, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.7027, + "full_recall_at_20": 0.5217, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.029, + "mean_recall_at_16000_tokens": 0.2027, + "mean_precision_at_10": 0.0841, + "zero_hit_rate": 0.1449, + "median_tokens_delivered": 66836, + "median_latency_ms": 50.03, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 39, + "mean_recall_at_5": 0.7329, + "full_recall_at_5": 0.5385, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2436, + "mean_recall_at_16000_tokens": 0.7244, + "mean_precision_at_10": 0.1859, + "zero_hit_rate": 0, + "median_tokens_delivered": 56450, + "median_latency_ms": 48.287, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.4488, + "full_recall_at_5": 0.2222, + "mean_recall_at_10": 0.6093, + "full_recall_at_10": 0.3889, + "mean_recall_at_20": 0.8346, + "full_recall_at_20": 0.6852, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1574, + "mean_recall_at_16000_tokens": 0.4333, + "mean_precision_at_10": 0.1241, + "zero_hit_rate": 0.037, + "median_tokens_delivered": 63529.5, + "median_latency_ms": 50.105, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.375, + "full_recall_at_5": 0.1667, + "mean_recall_at_10": 0.5525, + "full_recall_at_10": 0.3333, + "mean_recall_at_20": 0.7855, + "full_recall_at_20": 0.7037, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0556, + "mean_recall_at_16000_tokens": 0.3488, + "mean_precision_at_10": 0.1176, + "zero_hit_rate": 0.1481, + "median_tokens_delivered": 61941.5, + "median_latency_ms": 47.553, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 16856.4, + "peak_rss_mb": 18255.1, + "delta_rss_mb": 1398.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "keyword-search", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 52061, + "latency_ms": 65.86, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 72721, + "latency_ms": 51.302, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 77391, + "latency_ms": 69.017, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53804, + "latency_ms": 45.51, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 67858, + "latency_ms": 68.275, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 155510, + "latency_ms": 72.894, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 51927, + "latency_ms": 97.698, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 44754, + "latency_ms": 44.297, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87223, + "latency_ms": 82.713, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 45358, + "latency_ms": 38.736, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 102374, + "latency_ms": 40.394, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 39464, + "latency_ms": 43.821, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/normalize-arguments.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 40333, + "latency_ms": 39.774, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 45032, + "latency_ms": 26.765, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 64066, + "latency_ms": 67.411, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 84841, + "latency_ms": 76.255, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 70344, + "latency_ms": 78.505, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 65691, + "latency_ms": 71.667, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 84000, + "latency_ms": 64.063, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 45399, + "latency_ms": 48.953, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 73783, + "latency_ms": 53.51, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 46166, + "latency_ms": 57.333, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 80402, + "latency_ms": 68.019, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 39002, + "latency_ms": 27.199, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 75335, + "latency_ms": 39.208, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 46583, + "latency_ms": 45.507, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 60881, + "latency_ms": 51.142, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 41278, + "latency_ms": 45.601, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 62231, + "latency_ms": 45.813, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 63688, + "latency_ms": 34.633, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 46609, + "latency_ms": 42.021, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 61496, + "latency_ms": 51.528, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 35851, + "latency_ms": 32.804, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 96333, + "latency_ms": 50.03, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62246, + "latency_ms": 22.328, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 99388, + "latency_ms": 71.498, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 87406, + "latency_ms": 69.374, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 48035, + "latency_ms": 56.769, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [ + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 4, + "tokens_delivered": 23953, + "latency_ms": 34.769, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 143218, + "latency_ms": 48.287, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 63099, + "latency_ms": 35.583, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 83537, + "latency_ms": 85.153, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 89231, + "latency_ms": 43.217, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 28709, + "latency_ms": 22.812, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 44085, + "latency_ms": 38.45, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53523, + "latency_ms": 29.56, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 78960, + "latency_ms": 50.753, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 86165, + "latency_ms": 84.285, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 85215, + "latency_ms": 62.844, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 59030, + "latency_ms": 33.516, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 80040, + "latency_ms": 55.14, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 48574, + "latency_ms": 67.078, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 137075, + "latency_ms": 61.962, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 63528, + "latency_ms": 40.077, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 75580, + "latency_ms": 62.138, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 58669, + "latency_ms": 44.472, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 46865, + "latency_ms": 49.566, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 35083, + "latency_ms": 29.344, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 35641, + "latency_ms": 33.158, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 66786, + "latency_ms": 43.767, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 49650, + "latency_ms": 57.915, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 52150, + "latency_ms": 45.395, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 36292, + "latency_ms": 43.451, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 72027, + "latency_ms": 47.051, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 48302, + "latency_ms": 53.532, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 83669, + "latency_ms": 30.357, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38479, + "latency_ms": 41.302, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32619, + "latency_ms": 29.744, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 57707, + "latency_ms": 55.567, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 61601, + "latency_ms": 43.414, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 70058, + "latency_ms": 50.032, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 77408, + "latency_ms": 53.887, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 73667, + "latency_ms": 56.627, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 66836, + "latency_ms": 42.58, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 32327, + "latency_ms": 35.845, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 145671, + "latency_ms": 51.204, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48778, + "latency_ms": 28.188, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 151401, + "latency_ms": 41.46, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 52242, + "latency_ms": 40.601, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28401, + "latency_ms": 25.203, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 87615, + "latency_ms": 48.451, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34148, + "latency_ms": 39.626, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 56450, + "latency_ms": 34.606, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 53717, + "latency_ms": 50.178, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 44547, + "latency_ms": 46.586, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 67602, + "latency_ms": 50.064, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 76447, + "latency_ms": 68.162, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 77055, + "latency_ms": 59.911, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 61652, + "latency_ms": 37.2, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 48300, + "latency_ms": 50.93, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 45872, + "latency_ms": 82.231, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 79869, + "latency_ms": 86.002, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 78365, + "latency_ms": 83.835, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33797, + "latency_ms": 39.431, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 50056, + "latency_ms": 45.581, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 90953, + "latency_ms": 239.357, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 60512, + "latency_ms": 96.258, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 34528, + "latency_ms": 53.16, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 82575, + "latency_ms": 98.695, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 83755, + "latency_ms": 105.577, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 68966, + "latency_ms": 46.819, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37763, + "latency_ms": 37.312, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 90708, + "latency_ms": 66.936, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 56246, + "latency_ms": 28.982, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 63960, + "latency_ms": 42.563, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 85642, + "latency_ms": 89.381, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 86911, + "latency_ms": 63.533, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "test/promise.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 67322, + "latency_ms": 71.896, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/random-code-files.json b/benchmarks/context-retrieval/results/full-field-got/random-code-files.json new file mode 100644 index 00000000..5a0aeb78 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/random-code-files.json @@ -0,0 +1,3148 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-code-files", + "query": "Fix encoding with json `responseType` (#1996)", + "revision": "dbbd31777b3299ceb0db99798453e01ab8e597b1", + "returned": [ + "source/core/response.ts", + "source/core/options.ts", + "test/response-parse.ts", + "test/url-to-options.ts", + "source/as-promise/types.ts" + ] + }, + { + "provider_id": "random-code-files", + "query": "Fix for sending files with size `0` on `stat` (#1488)", + "revision": "6aa86f2494194907f6c6c8b4774dfa1f69df6876", + "returned": [ + "test/types/create-test-server/index.d.ts", + "test/agent.ts", + "test/pagination.ts", + "source/core/utils/unhandle.ts", + "source/core/utils/get-buffer.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "random-code-files", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.0901, + "full_recall_at_5": 0.0278, + "mean_recall_at_10": 0.1918, + "full_recall_at_10": 0.0833, + "mean_recall_at_20": 0.3176, + "full_recall_at_20": 0.1481, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0448, + "mean_recall_at_16000_tokens": 0.181, + "mean_precision_at_10": 0.037, + "zero_hit_rate": 0.4907, + "median_tokens_delivered": 31269.5, + "median_latency_ms": 16.9115, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 99, + "mean_recall_at_5": 0.063, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1183, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2556, + "full_recall_at_20": 0.0707, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0337, + "mean_recall_at_16000_tokens": 0.1116, + "mean_precision_at_10": 0.0303, + "zero_hit_rate": 0.5354, + "median_tokens_delivered": 31271, + "median_latency_ms": 17.069, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 9, + "mean_recall_at_5": 0.3889, + "full_recall_at_5": 0.3333, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1667, + "mean_recall_at_16000_tokens": 0.9444, + "mean_precision_at_10": 0.1111, + "zero_hit_rate": 0, + "median_tokens_delivered": 30414, + "median_latency_ms": 16.676, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.1062, + "full_recall_at_5": 0.0185, + "mean_recall_at_10": 0.1969, + "full_recall_at_10": 0.0741, + "mean_recall_at_20": 0.3142, + "full_recall_at_20": 0.1481, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0741, + "mean_recall_at_16000_tokens": 0.1877, + "mean_precision_at_10": 0.0407, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 30355.5, + "median_latency_ms": 16.8015, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.0741, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.1867, + "full_recall_at_10": 0.0926, + "mean_recall_at_20": 0.321, + "full_recall_at_20": 0.1481, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0154, + "mean_recall_at_16000_tokens": 0.1744, + "mean_precision_at_10": 0.0333, + "zero_hit_rate": 0.4815, + "median_tokens_delivered": 31734.5, + "median_latency_ms": 17.022, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 16696.5, + "peak_rss_mb": 17212.2, + "delta_rss_mb": 515.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-code-files", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27476, + "latency_ms": 36.616, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 39306, + "latency_ms": 22.163, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 28287, + "latency_ms": 20.91, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31268, + "latency_ms": 18.382, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 50945, + "latency_ms": 16.718, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 89425, + "latency_ms": 16.956, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 35126, + "latency_ms": 16.738, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 49042, + "latency_ms": 24.848, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 26485, + "latency_ms": 40.814, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 44707, + "latency_ms": 17.143, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22538, + "latency_ms": 17.33, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29564, + "latency_ms": 16.365, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 35775, + "latency_ms": 19.552, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47548, + "latency_ms": 20.058, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38902, + "latency_ms": 18.744, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36258, + "latency_ms": 19.381, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34821, + "latency_ms": 26.548, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 37290, + "latency_ms": 20.379, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32856, + "latency_ms": 18.102, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34816, + "latency_ms": 17.918, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28804, + "latency_ms": 17.572, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 33739, + "latency_ms": 18.36, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 42300, + "latency_ms": 29.903, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39695, + "latency_ms": 30.999, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 31571, + "latency_ms": 23.113, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27687, + "latency_ms": 19.475, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22150, + "latency_ms": 19.884, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 32098, + "latency_ms": 19.306, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31371, + "latency_ms": 17.396, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 40945, + "latency_ms": 17.348, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 9, + "tokens_delivered": 26123, + "latency_ms": 17.202, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23360, + "latency_ms": 16.867, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 31458, + "latency_ms": 19.32, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 54591, + "latency_ms": 15.289, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17159, + "latency_ms": 16.86, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/types.ts" + ], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 42078, + "latency_ms": 15.388, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 30479, + "latency_ms": 15.551, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "test/promise.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 16753, + "latency_ms": 18.965, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18287, + "latency_ms": 21.726, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 81738, + "latency_ms": 18.416, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 29097, + "latency_ms": 17.644, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 44357, + "latency_ms": 16.794, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45723, + "latency_ms": 16.021, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29798, + "latency_ms": 18.427, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 25461, + "latency_ms": 15.573, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 30414, + "latency_ms": 16.288, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 58847, + "latency_ms": 15.682, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 49858, + "latency_ms": 15.733, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18765, + "latency_ms": 16.344, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/error.ts", + "test/helpers.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 2, + "tokens_delivered": 40405, + "latency_ms": 16.239, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 26008, + "latency_ms": 14.607, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21693, + "latency_ms": 15.957, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 52042, + "latency_ms": 16.263, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15898, + "latency_ms": 15.734, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts" + ], + "missed": [ + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 28071, + "latency_ms": 18.436, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/utils/get-body-size.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 26674, + "latency_ms": 21.178, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20021, + "latency_ms": 17.749, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22389, + "latency_ms": 19.063, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 32883, + "latency_ms": 18.441, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "test/redirects.ts" + ], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 10, + "tokens_delivered": 30240, + "latency_ms": 16.477, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21224, + "latency_ms": 17.251, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 33599, + "latency_ms": 14.976, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 26412, + "latency_ms": 17.069, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21705, + "latency_ms": 17.086, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27255, + "latency_ms": 15.738, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35226, + "latency_ms": 16.737, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35237, + "latency_ms": 14.442, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 30232, + "latency_ms": 16.77, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 20789, + "latency_ms": 15.712, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36140, + "latency_ms": 15.333, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 31963, + "latency_ms": 13.386, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42533, + "latency_ms": 16.173, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42368, + "latency_ms": 19.569, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 41926, + "latency_ms": 20.01, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 21382, + "latency_ms": 17.485, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "test/early-307-redirects.ts" + ], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 43257, + "latency_ms": 17.542, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 36933, + "latency_ms": 16.676, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 60386, + "latency_ms": 18.508, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 41160, + "latency_ms": 14.723, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 33055, + "latency_ms": 16.308, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21305, + "latency_ms": 15.518, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 28788, + "latency_ms": 15.774, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28345, + "latency_ms": 17.188, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25582, + "latency_ms": 14.439, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24959, + "latency_ms": 15.567, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 27042, + "latency_ms": 14.249, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40721, + "latency_ms": 15.216, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22195, + "latency_ms": 16.325, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "test/headers.ts" + ], + "missed": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 23345, + "latency_ms": 15.844, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 18036, + "latency_ms": 16.062, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15655, + "latency_ms": 19.438, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28387, + "latency_ms": 19.4, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 35770, + "latency_ms": 16.833, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38132, + "latency_ms": 21.44, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24669, + "latency_ms": 16.975, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45967, + "latency_ms": 17.088, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24497, + "latency_ms": 13.778, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19716, + "latency_ms": 16.58, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "source/core/index.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 29993, + "latency_ms": 16.207, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37841, + "latency_ms": 14.165, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 55427, + "latency_ms": 15.73, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34957, + "latency_ms": 14.883, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30631, + "latency_ms": 16.057, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27711, + "latency_ms": 14.229, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28673, + "latency_ms": 15.593, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/timeout.ts" + ], + "missed": [ + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 7, + "tokens_delivered": 31271, + "latency_ms": 15.895, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30511, + "latency_ms": 15.958, + "indexed_revision_matches": true + }, + { + "provider_id": "random-code-files", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 35607, + "latency_ms": 18.747, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/random-files.json b/benchmarks/context-retrieval/results/full-field-got/random-files.json new file mode 100644 index 00000000..c2accfaa --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/random-files.json @@ -0,0 +1,3140 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "Fix false `ReadError` on responses without `Content-Length`", + "revision": "b03e6f95ebb5fe3c51512fe0190a20385e2f9ee9", + "returned": [ + "test/types/create-test-server/index.d.ts", + ".github/ISSUE_TEMPLATE/1-bug-report.md", + ".gitattributes", + "documentation/tips.md", + "test/headers.ts" + ] + }, + { + "provider_id": "random-files", + "query": "Fix hanging promise on timeout on HTTP error", + "revision": "7582d916db72f7bab2b2c162d37cfcfde08bbba7", + "returned": [ + "test/retry.ts", + "test/headers.ts", + "test/helpers/with-server.ts", + "test/unix-socket.ts", + ".github/funding.yml" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.0525, + "full_recall_at_5": 0.0278, + "mean_recall_at_10": 0.113, + "full_recall_at_10": 0.0463, + "mean_recall_at_20": 0.2093, + "full_recall_at_20": 0.0833, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0069, + "mean_recall_at_16000_tokens": 0.091, + "mean_precision_at_10": 0.0222, + "zero_hit_rate": 0.6296, + "median_tokens_delivered": 33616.5, + "median_latency_ms": 16.862, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 103, + "mean_recall_at_5": 0.0259, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0699, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1709, + "full_recall_at_20": 0.0388, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0073, + "mean_recall_at_16000_tokens": 0.0663, + "mean_precision_at_10": 0.0175, + "zero_hit_rate": 0.6602, + "median_tokens_delivered": 33221, + "median_latency_ms": 16.856, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 5, + "mean_recall_at_5": 0.6, + "full_recall_at_5": 0.6, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.6, + "mean_precision_at_10": 0.12, + "zero_hit_rate": 0, + "median_tokens_delivered": 40763, + "median_latency_ms": 16.868, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.0401, + "full_recall_at_5": 0.0185, + "mean_recall_at_10": 0.1179, + "full_recall_at_10": 0.037, + "mean_recall_at_20": 0.1716, + "full_recall_at_20": 0.0556, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0679, + "mean_precision_at_10": 0.0222, + "zero_hit_rate": 0.6852, + "median_tokens_delivered": 38025.5, + "median_latency_ms": 16.748, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.0648, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.108, + "full_recall_at_10": 0.0556, + "mean_recall_at_20": 0.2469, + "full_recall_at_20": 0.1111, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0139, + "mean_recall_at_16000_tokens": 0.1142, + "mean_precision_at_10": 0.0222, + "zero_hit_rate": 0.5741, + "median_tokens_delivered": 30902, + "median_latency_ms": 17.0545, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 16663.1, + "peak_rss_mb": 17218.3, + "delta_rss_mb": 555.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "test/create.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 128312, + "latency_ms": 36.638, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 23457, + "latency_ms": 22.12, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20101, + "latency_ms": 23.042, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 115170, + "latency_ms": 17.204, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 25749, + "latency_ms": 16.459, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 84902, + "latency_ms": 16.531, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46345, + "latency_ms": 16.709, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35753, + "latency_ms": 21.894, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38711, + "latency_ms": 30.834, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25476, + "latency_ms": 20.424, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73946, + "latency_ms": 17.99, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 143922, + "latency_ms": 17.281, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 40459, + "latency_ms": 18.305, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26819, + "latency_ms": 21.622, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 33469, + "latency_ms": 17.222, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123066, + "latency_ms": 18.572, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 33608, + "latency_ms": 22.494, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21599, + "latency_ms": 25.409, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 147848, + "latency_ms": 18.534, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31764, + "latency_ms": 17.944, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32424, + "latency_ms": 16.928, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 110215, + "latency_ms": 19.712, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26745, + "latency_ms": 29.192, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 123106, + "latency_ms": 39.541, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47662, + "latency_ms": 23.591, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "tsconfig.json" + ], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 138468, + "latency_ms": 19.193, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45260, + "latency_ms": 20.028, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 26930, + "latency_ms": 20.045, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 35292, + "latency_ms": 16.735, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12722, + "latency_ms": 17.306, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24262, + "latency_ms": 18.585, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 24415, + "latency_ms": 17.138, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 116504, + "latency_ms": 17.796, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15390, + "latency_ms": 14.957, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39619, + "latency_ms": 17.123, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 121149, + "latency_ms": 15.672, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 47724, + "latency_ms": 15.235, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 21750, + "latency_ms": 18.446, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 17724, + "latency_ms": 21.731, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 76479, + "latency_ms": 18.039, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 134112, + "latency_ms": 16.868, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22890, + "latency_ms": 16.59, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35505, + "latency_ms": 19.409, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15910, + "latency_ms": 16.284, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 30783, + "latency_ms": 16.856, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39530, + "latency_ms": 15.469, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27998, + "latency_ms": 16.356, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 6, + "tokens_delivered": 44206, + "latency_ms": 15.023, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37952, + "latency_ms": 17.225, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/helpers.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/error.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 32929, + "latency_ms": 15.39, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 132351, + "latency_ms": 15.181, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/promise.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 142218, + "latency_ms": 15.119, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44448, + "latency_ms": 16.378, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24486, + "latency_ms": 15.954, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "test/retry.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 49714, + "latency_ms": 18.31, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 108592, + "latency_ms": 21.25, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16188, + "latency_ms": 18.218, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 28462, + "latency_ms": 17.571, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30578, + "latency_ms": 19.261, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28166, + "latency_ms": 16.838, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12728, + "latency_ms": 16.986, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10505, + "latency_ms": 14.932, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20125, + "latency_ms": 17.503, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26378, + "latency_ms": 16.714, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 19330, + "latency_ms": 16.455, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26010, + "latency_ms": 15.024, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 116589, + "latency_ms": 15.796, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 39134, + "latency_ms": 15.885, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105294, + "latency_ms": 15.487, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 20529, + "latency_ms": 14.815, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20103, + "latency_ms": 14.356, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16040, + "latency_ms": 16.104, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 14, + "tokens_delivered": 22398, + "latency_ms": 19.024, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 47619, + "latency_ms": 20.444, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 17924, + "latency_ms": 19.989, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36329, + "latency_ms": 18.517, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21416, + "latency_ms": 17.675, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49100, + "latency_ms": 15.694, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28562, + "latency_ms": 14.484, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33625, + "latency_ms": 15.292, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 117459, + "latency_ms": 14.7, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 40763, + "latency_ms": 15.712, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21073, + "latency_ms": 18.182, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32320, + "latency_ms": 15.936, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 11886, + "latency_ms": 15.236, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 37678, + "latency_ms": 13.068, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 46844, + "latency_ms": 15.328, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26967, + "latency_ms": 14.642, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "test/headers.ts" + ], + "missed": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 14, + "tokens_delivered": 15714, + "latency_ms": 15.306, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 31009, + "latency_ms": 16.333, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 118966, + "latency_ms": 16.782, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28922, + "latency_ms": 20.396, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 23071, + "latency_ms": 17.268, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 113610, + "latency_ms": 18.302, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38099, + "latency_ms": 16.072, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 33221, + "latency_ms": 17.338, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 40387, + "latency_ms": 14.752, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 126712, + "latency_ms": 15.846, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25898, + "latency_ms": 14.633, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35905, + "latency_ms": 15.334, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20078, + "latency_ms": 15.097, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 109765, + "latency_ms": 15.065, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23631, + "latency_ms": 14.741, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "test/cache.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 36326, + "latency_ms": 15.802, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 131366, + "latency_ms": 14.326, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts" + ], + "missed": [ + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 42508, + "latency_ms": 16.113, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30795, + "latency_ms": 16.157, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 121830, + "latency_ms": 17.137, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/repomix-compressed.json b/benchmarks/context-retrieval/results/full-field-got/repomix-compressed.json new file mode 100644 index 00000000..4afb6b9d --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/repomix-compressed.json @@ -0,0 +1,3093 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [ + { + "provider_id": "repomix-compressed", + "why": "scored exactly zero — read the raw payload before believing it" + } + ], + "nominated_for_hand_audit": [], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "repomix-compressed", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0687, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 82501.5, + "median_latency_ms": 1182.0755, + "payload_kind": [ + "whole-repository" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 108, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0687, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 82501.5, + "median_latency_ms": 1182.0755, + "payload_kind": [ + "whole-repository" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0741, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 82503, + "median_latency_ms": 1166.815, + "payload_kind": [ + "whole-repository" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0633, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 81589, + "median_latency_ms": 1192.5925, + "payload_kind": [ + "whole-repository" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 19704.5, + "peak_rss_mb": 21679.5, + "delta_rss_mb": 1975, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "repomix-compressed", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 42537, + "latency_ms": 1617.282, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 69, + "tokens_delivered": 90110, + "latency_ms": 1323.746, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 83861, + "latency_ms": 1148.53, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 60953, + "latency_ms": 1198.309, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 52, + "tokens_delivered": 78839, + "latency_ms": 1156.609, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 103711, + "latency_ms": 1211.703, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 104, + "tokens_delivered": 88467, + "latency_ms": 1111.872, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 49, + "tokens_delivered": 72498, + "latency_ms": 1387.255, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 99736, + "latency_ms": 1139.798, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 48, + "tokens_delivered": 70756, + "latency_ms": 1230.244, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 103915, + "latency_ms": 1189.342, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 67, + "tokens_delivered": 60821, + "latency_ms": 1330.971, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 30, + "tokens_delivered": 37935, + "latency_ms": 1146.874, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 78, + "tokens_delivered": 64389, + "latency_ms": 1203.818, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 61112, + "latency_ms": 1182.117, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 46, + "tokens_delivered": 92254, + "latency_ms": 1178.706, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 61247, + "latency_ms": 1244.757, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 47, + "tokens_delivered": 70613, + "latency_ms": 1165.204, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 90571, + "latency_ms": 1141.733, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 51, + "tokens_delivered": 83716, + "latency_ms": 1310.572, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 88178, + "latency_ms": 1273.877, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 31, + "tokens_delivered": 39927, + "latency_ms": 1130, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 90460, + "latency_ms": 1785.855, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 87, + "tokens_delivered": 81073, + "latency_ms": 1297.808, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 96411, + "latency_ms": 1208.648, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json", + "tsconfig.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 44, + "tokens_delivered": 84064, + "latency_ms": 1148.546, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 84020, + "latency_ms": 1185.921, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 88478, + "latency_ms": 1321.304, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 84040, + "latency_ms": 1326.332, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 32, + "tokens_delivered": 63350, + "latency_ms": 1182.034, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 48, + "tokens_delivered": 71487, + "latency_ms": 1321.413, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 32, + "tokens_delivered": 65498, + "latency_ms": 1144.965, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 63, + "tokens_delivered": 38119, + "latency_ms": 1161.958, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 101019, + "latency_ms": 1194.23, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [ + "source/as-promise/types.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 65498, + "latency_ms": 1042.113, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 52, + "tokens_delivered": 101566, + "latency_ms": 1216.571, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 88365, + "latency_ms": 1129.846, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 29, + "tokens_delivered": 40972, + "latency_ms": 1164.417, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 30, + "tokens_delivered": 37919, + "latency_ms": 1229.095, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 54, + "tokens_delivered": 104191, + "latency_ms": 1190.955, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 48, + "tokens_delivered": 70761, + "latency_ms": 1326.448, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 46, + "tokens_delivered": 89714, + "latency_ms": 1174.637, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 64, + "tokens_delivered": 100800, + "latency_ms": 1263.705, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 42078, + "latency_ms": 1146.915, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 27, + "tokens_delivered": 41356, + "latency_ms": 1200.251, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 83386, + "latency_ms": 1062.572, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 51, + "tokens_delivered": 85685, + "latency_ms": 1054.841, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 89714, + "latency_ms": 1168.043, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 46, + "tokens_delivered": 90026, + "latency_ms": 1260.464, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts" + ], + "missed": [ + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 29, + "tokens_delivered": 60860, + "latency_ms": 1270.095, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 51, + "tokens_delivered": 84469, + "latency_ms": 1152.963, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 27, + "tokens_delivered": 40980, + "latency_ms": 1289.565, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 103938, + "latency_ms": 1198.902, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 60112, + "latency_ms": 1315.726, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 54, + "tokens_delivered": 83873, + "latency_ms": 1183.469, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 34, + "tokens_delivered": 64535, + "latency_ms": 1210.174, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 30, + "tokens_delivered": 37883, + "latency_ms": 1088.496, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 59977, + "latency_ms": 1299.946, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 39, + "tokens_delivered": 38119, + "latency_ms": 1195.801, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 32, + "tokens_delivered": 64420, + "latency_ms": 1226.95, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 31, + "tokens_delivered": 40986, + "latency_ms": 1250.516, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 31, + "tokens_delivered": 41091, + "latency_ms": 1159.733, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 30, + "tokens_delivered": 38090, + "latency_ms": 1145.905, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 84062, + "latency_ms": 1120.164, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 29, + "tokens_delivered": 40861, + "latency_ms": 982.476, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 93624, + "latency_ms": 1111.127, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [ + "test/progress.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 92, + "tokens_delivered": 79645, + "latency_ms": 1123.675, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 82, + "tokens_delivered": 70641, + "latency_ms": 1182.69, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 42463, + "latency_ms": 1311.376, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 84036, + "latency_ms": 1230.918, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 61140, + "latency_ms": 1395.351, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 87659, + "latency_ms": 2053.287, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 55, + "tokens_delivered": 87776, + "latency_ms": 1255.71, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 59932, + "latency_ms": 1140.137, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 31, + "tokens_delivered": 41197, + "latency_ms": 1142.449, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 104238, + "latency_ms": 1205.12, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 83504, + "latency_ms": 1070.341, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 103266, + "latency_ms": 1125.255, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 51, + "tokens_delivered": 83541, + "latency_ms": 1100.753, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 78374, + "latency_ms": 1090.832, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 93661, + "latency_ms": 994.269, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 84109, + "latency_ms": 1026.342, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 71, + "tokens_delivered": 81617, + "latency_ms": 1111.509, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 46, + "tokens_delivered": 83389, + "latency_ms": 1072.22, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 32, + "tokens_delivered": 64406, + "latency_ms": 1117.075, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 44, + "tokens_delivered": 71730, + "latency_ms": 1147.429, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 84245, + "latency_ms": 1050.425, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 85335, + "latency_ms": 1054.419, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 48, + "tokens_delivered": 71575, + "latency_ms": 1406.222, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 31, + "tokens_delivered": 39963, + "latency_ms": 1246.365, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 31, + "tokens_delivered": 41175, + "latency_ms": 1278.197, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 51, + "tokens_delivered": 83833, + "latency_ms": 1306.592, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 84456, + "latency_ms": 1113.4, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 44, + "tokens_delivered": 84559, + "latency_ms": 1160.434, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 42, + "tokens_delivered": 42381, + "latency_ms": 1131.383, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 90005, + "latency_ms": 1157.628, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 49, + "tokens_delivered": 71325, + "latency_ms": 1230.381, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 67, + "tokens_delivered": 41451, + "latency_ms": 1374.414, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 91543, + "latency_ms": 1478.554, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 86469, + "latency_ms": 1795.311, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 79792, + "latency_ms": 1038.62, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 45, + "tokens_delivered": 76562, + "latency_ms": 1066.565, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 46, + "tokens_delivered": 94135, + "latency_ms": 1084.322, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 84062, + "latency_ms": 1079.269, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "source/create.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 64, + "tokens_delivered": 84161, + "latency_ms": 1141.119, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 70, + "tokens_delivered": 101156, + "latency_ms": 1169.213, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 88335, + "latency_ms": 1211.499, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-compressed", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 27, + "tokens_delivered": 64534, + "latency_ms": 1153.117, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/repomix-pack-all.json b/benchmarks/context-retrieval/results/full-field-got/repomix-pack-all.json new file mode 100644 index 00000000..852a041a --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/repomix-pack-all.json @@ -0,0 +1,3093 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [ + { + "provider_id": "repomix-pack-all", + "why": "scored exactly zero — read the raw payload before believing it" + } + ], + "nominated_for_hand_audit": [], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "repomix-pack-all", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 163443.5, + "median_latency_ms": 1069.0345, + "payload_kind": [ + "whole-repository" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 108, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 163443.5, + "median_latency_ms": 1069.0345, + "payload_kind": [ + "whole-repository" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 0, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 0, + "median_latency_ms": 0, + "payload_kind": [], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 163443.5, + "median_latency_ms": 1066.0765, + "payload_kind": [ + "whole-repository" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 0, + "median_tokens_delivered": 161194.5, + "median_latency_ms": 1073.684, + "payload_kind": [ + "whole-repository" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 19707.3, + "peak_rss_mb": 21267, + "delta_rss_mb": 1559.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "repomix-pack-all", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 110684, + "latency_ms": 1522.877, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 69, + "tokens_delivered": 181365, + "latency_ms": 1178.711, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 168975, + "latency_ms": 1101.909, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 134605, + "latency_ms": 1045.21, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 52, + "tokens_delivered": 157133, + "latency_ms": 1273.354, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 278639, + "latency_ms": 1095.793, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 104, + "tokens_delivered": 178881, + "latency_ms": 1037.883, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 49, + "tokens_delivered": 150846, + "latency_ms": 959.804, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 203930, + "latency_ms": 1011.083, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 48, + "tokens_delivered": 146936, + "latency_ms": 1052.791, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 279222, + "latency_ms": 1050.584, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 67, + "tokens_delivered": 134380, + "latency_ms": 1301.489, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 96708, + "latency_ms": 1060.071, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 78, + "tokens_delivered": 127782, + "latency_ms": 1030.942, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 134776, + "latency_ms": 1091.418, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 46, + "tokens_delivered": 185279, + "latency_ms": 1000.705, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 135963, + "latency_ms": 1105.813, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 47, + "tokens_delivered": 146828, + "latency_ms": 1050.241, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 182930, + "latency_ms": 1077.126, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 51, + "tokens_delivered": 165053, + "latency_ms": 991.213, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 177996, + "latency_ms": 1178.502, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 103240, + "latency_ms": 1120.709, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 182766, + "latency_ms": 1095.418, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 87, + "tokens_delivered": 159715, + "latency_ms": 1023.977, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 196694, + "latency_ms": 1343.182, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json", + "tsconfig.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 44, + "tokens_delivered": 165716, + "latency_ms": 1504.594, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 165672, + "latency_ms": 1113.105, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 178951, + "latency_ms": 1054.953, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 165442, + "latency_ms": 1085.956, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 32, + "tokens_delivered": 125623, + "latency_ms": 1084.304, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 48, + "tokens_delivered": 148182, + "latency_ms": 1213.638, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 32, + "tokens_delivered": 130569, + "latency_ms": 1309.701, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 63, + "tokens_delivered": 99239, + "latency_ms": 1188.598, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 209772, + "latency_ms": 1071.291, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [ + "source/as-promise/types.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 130761, + "latency_ms": 1015.22, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 52, + "tokens_delivered": 211360, + "latency_ms": 1000.178, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 178456, + "latency_ms": 1094.074, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 29, + "tokens_delivered": 105853, + "latency_ms": 1010.484, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 97288, + "latency_ms": 1035.779, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 54, + "tokens_delivered": 280784, + "latency_ms": 1203.716, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 48, + "tokens_delivered": 147121, + "latency_ms": 1066.778, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 46, + "tokens_delivered": 180857, + "latency_ms": 1079.477, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 64, + "tokens_delivered": 209203, + "latency_ms": 1065.743, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 108851, + "latency_ms": 1031.077, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 27, + "tokens_delivered": 107189, + "latency_ms": 1183.277, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 166078, + "latency_ms": 1020.567, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 51, + "tokens_delivered": 173268, + "latency_ms": 1129.027, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 181085, + "latency_ms": 1058.463, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 46, + "tokens_delivered": 181686, + "latency_ms": 1093.114, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts" + ], + "missed": [ + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 29, + "tokens_delivered": 134429, + "latency_ms": 1065.375, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 51, + "tokens_delivered": 170483, + "latency_ms": 942.428, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 27, + "tokens_delivered": 105996, + "latency_ms": 983.322, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 279502, + "latency_ms": 1106.746, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 133033, + "latency_ms": 1129.531, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 54, + "tokens_delivered": 168961, + "latency_ms": 1077.803, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 34, + "tokens_delivered": 128529, + "latency_ms": 1177.724, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 96490, + "latency_ms": 1149.367, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 132196, + "latency_ms": 1052.464, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 39, + "tokens_delivered": 99217, + "latency_ms": 1117.357, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 32, + "tokens_delivered": 127833, + "latency_ms": 1057.419, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 106116, + "latency_ms": 1128.389, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 106560, + "latency_ms": 1076.077, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 98746, + "latency_ms": 1090.241, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 168805, + "latency_ms": 1044.89, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 29, + "tokens_delivered": 105608, + "latency_ms": 1007.78, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 189350, + "latency_ms": 1061.858, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [ + "test/progress.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 92, + "tokens_delivered": 155904, + "latency_ms": 1094.037, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 82, + "tokens_delivered": 147015, + "latency_ms": 1118.209, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 110681, + "latency_ms": 1033.7, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 168738, + "latency_ms": 1066.756, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 135594, + "latency_ms": 989.916, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 176771, + "latency_ms": 977.647, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 55, + "tokens_delivered": 177136, + "latency_ms": 1044.487, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 30, + "tokens_delivered": 131959, + "latency_ms": 1002.125, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 106858, + "latency_ms": 995.373, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 281110, + "latency_ms": 1162.84, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 166191, + "latency_ms": 1154.965, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 53, + "tokens_delivered": 275248, + "latency_ms": 1153.522, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 51, + "tokens_delivered": 164582, + "latency_ms": 2045.586, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 156728, + "latency_ms": 1136.981, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 189813, + "latency_ms": 1147.402, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 169196, + "latency_ms": 1082.126, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 71, + "tokens_delivered": 162305, + "latency_ms": 1056.771, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 46, + "tokens_delivered": 166693, + "latency_ms": 1047.372, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 32, + "tokens_delivered": 127799, + "latency_ms": 985.368, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 44, + "tokens_delivered": 149288, + "latency_ms": 1015.51, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 169508, + "latency_ms": 1023.863, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 172574, + "latency_ms": 976.276, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 48, + "tokens_delivered": 148473, + "latency_ms": 968.886, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 103266, + "latency_ms": 869.82, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 31, + "tokens_delivered": 106662, + "latency_ms": 970.286, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 51, + "tokens_delivered": 168947, + "latency_ms": 1048.751, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 170235, + "latency_ms": 1022.271, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 44, + "tokens_delivered": 169190, + "latency_ms": 979.476, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 42, + "tokens_delivered": 109354, + "latency_ms": 1012.853, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 181524, + "latency_ms": 975.764, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 49, + "tokens_delivered": 148936, + "latency_ms": 1097.999, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 67, + "tokens_delivered": 107675, + "latency_ms": 1249.505, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 184068, + "latency_ms": 1333.396, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 174705, + "latency_ms": 1011.761, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 157336, + "latency_ms": 1381.705, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 45, + "tokens_delivered": 155310, + "latency_ms": 1300.264, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 46, + "tokens_delivered": 192241, + "latency_ms": 1097.893, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 168751, + "latency_ms": 1033.196, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "source/create.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 64, + "tokens_delivered": 168798, + "latency_ms": 1064.453, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 70, + "tokens_delivered": 209981, + "latency_ms": 1209.305, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 50, + "tokens_delivered": 178168, + "latency_ms": 1161.372, + "indexed_revision_matches": true + }, + { + "provider_id": "repomix-pack-all", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 27, + "tokens_delivered": 129107, + "latency_ms": 1106.642, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/repowise.json b/benchmarks/context-retrieval/results/full-field-got/repowise.json new file mode 100644 index 00000000..55d56ff5 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/repowise.json @@ -0,0 +1,3164 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "repowise", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "source/core/utils/dns-ip-version.ts", + "source/core/index.ts", + "source/as-promise/core.ts", + "source/as-promise/index.ts", + "source/index.ts" + ] + }, + { + "provider_id": "repowise", + "query": "Fix unhandled `The server aborted pending request` rejection", + "revision": "04f3ea4b7995e1609555dd13f00641b8f470de4c", + "returned": [ + "source/core/index.ts", + "source/as-promise/core.ts", + "test/helpers/with-server.ts", + "source/types.ts", + "source/as-promise/create-rejection.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "repowise", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.4136, + "full_recall_at_5": 0.1574, + "mean_recall_at_10": 0.5427, + "full_recall_at_10": 0.25, + "mean_recall_at_20": 0.6346, + "full_recall_at_20": 0.3796, + "mean_recall_at_1000_tokens": 0.4606, + "mean_recall_at_4000_tokens": 0.6346, + "mean_recall_at_16000_tokens": 0.6346, + "mean_precision_at_10": 0.1088, + "zero_hit_rate": 0.1204, + "median_tokens_delivered": 2022.5, + "median_latency_ms": 40075.505, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 81, + "mean_recall_at_5": 0.3107, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.3903, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.5128, + "full_recall_at_20": 0.1728, + "mean_recall_at_1000_tokens": 0.3302, + "mean_recall_at_4000_tokens": 0.5128, + "mean_recall_at_16000_tokens": 0.5128, + "mean_precision_at_10": 0.0969, + "zero_hit_rate": 0.1605, + "median_tokens_delivered": 2033, + "median_latency_ms": 39673.445, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 27, + "mean_recall_at_5": 0.7222, + "full_recall_at_5": 0.6296, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.8519, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1445, + "zero_hit_rate": 0, + "median_tokens_delivered": 1999, + "median_latency_ms": 41410.378, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3889, + "full_recall_at_5": 0.1296, + "mean_recall_at_10": 0.5253, + "full_recall_at_10": 0.2037, + "mean_recall_at_20": 0.6642, + "full_recall_at_20": 0.3889, + "mean_recall_at_1000_tokens": 0.4599, + "mean_recall_at_4000_tokens": 0.6642, + "mean_recall_at_16000_tokens": 0.6642, + "mean_precision_at_10": 0.1078, + "zero_hit_rate": 0.0926, + "median_tokens_delivered": 2045, + "median_latency_ms": 39214.198, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.4383, + "full_recall_at_5": 0.1852, + "mean_recall_at_10": 0.5602, + "full_recall_at_10": 0.2963, + "mean_recall_at_20": 0.6049, + "full_recall_at_20": 0.3704, + "mean_recall_at_1000_tokens": 0.4614, + "mean_recall_at_4000_tokens": 0.6049, + "mean_recall_at_16000_tokens": 0.6049, + "mean_precision_at_10": 0.1098, + "zero_hit_rate": 0.1481, + "median_tokens_delivered": 2007.5, + "median_latency_ms": 41914.162, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 20851.3, + "peak_rss_mb": 43295.8, + "delta_rss_mb": 22444.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 589824 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "repowise", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2014, + "latency_ms": 45921.806, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2128, + "latency_ms": 48160.961, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2023, + "latency_ms": 65061.402, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1983, + "latency_ms": 73044.815, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2048, + "latency_ms": 64796.448, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/timed-out.ts" + ], + "missed": [ + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2001, + "latency_ms": 64288.956, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 2157, + "latency_ms": 50177.872, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1861, + "latency_ms": 47831.351, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2056, + "latency_ms": 49508.326, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1233, + "latency_ms": 55476.144, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1978, + "latency_ms": 62936.928, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2145, + "latency_ms": 58853.518, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 1916, + "latency_ms": 50199.548, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1912, + "latency_ms": 74719.832, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2061, + "latency_ms": 48570.346, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2049, + "latency_ms": 57473.54, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1966, + "latency_ms": 51269.932, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2000, + "latency_ms": 47072.623, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 1991, + "latency_ms": 49918.316, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 2115, + "latency_ms": 48702.73, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2041, + "latency_ms": 47183.808, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2046, + "latency_ms": 50834.906, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2290, + "latency_ms": 52386.422, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2152, + "latency_ms": 48992.457, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1959, + "latency_ms": 51761.91, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1666, + "latency_ms": 46304.433, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2078, + "latency_ms": 46279.167, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2029, + "latency_ms": 48084.266, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1997, + "latency_ms": 48985.443, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2009, + "latency_ms": 45587.257, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.2143, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 4, + "tokens_delivered": 1814, + "latency_ms": 44298.186, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2020, + "latency_ms": 42658.147, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1992, + "latency_ms": 45585.551, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.125, + "recall_at_20": 0.3333, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 2093, + "latency_ms": 49160.477, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1976, + "latency_ms": 43333.427, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 2096, + "latency_ms": 44383.372, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2075, + "latency_ms": 44247.09, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2053, + "latency_ms": 40927.984, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 6, + "tokens_delivered": 2015, + "latency_ms": 40197.02, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 2146, + "latency_ms": 47281.481, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2022, + "latency_ms": 41410.378, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2036, + "latency_ms": 42786.888, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1987, + "latency_ms": 42880.651, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1971, + "latency_ms": 39486.759, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 10, + "tokens_delivered": 2129, + "latency_ms": 39673.445, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 2146, + "latency_ms": 39953.99, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2179, + "latency_ms": 41401.171, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 2019, + "latency_ms": 45767.389, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 2006, + "latency_ms": 40446.805, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "source/core/errors.ts" + ], + "missed": [ + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.2, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 6, + "tokens_delivered": 1998, + "latency_ms": 40217.866, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2049, + "latency_ms": 40298.366, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts" + ], + "missed": [ + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2222, + "recall_at_20": 0.5, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2091, + "latency_ms": 38667.926, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2005, + "latency_ms": 46244.56, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1915, + "latency_ms": 41170.177, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 5, + "tokens_delivered": 2007, + "latency_ms": 43422.923, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1759, + "latency_ms": 43621.985, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2055, + "latency_ms": 38473.63, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 2088, + "latency_ms": 38938.592, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1702, + "latency_ms": 38986.197, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1992, + "latency_ms": 41011.56, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 2070, + "latency_ms": 35921.222, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1429, + "recall_at_20": 0.3333, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 2040, + "latency_ms": 37385.852, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2061, + "latency_ms": 37258.663, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 14, + "tokens_delivered": 2202, + "latency_ms": 37506.871, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 2033, + "latency_ms": 37034.544, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 2041, + "latency_ms": 37492.886, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [ + "test/progress.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 16, + "tokens_delivered": 2170, + "latency_ms": 37993.353, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2078, + "latency_ms": 37520.77, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2062, + "latency_ms": 36065.936, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1895, + "latency_ms": 38089.669, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1968, + "latency_ms": 38480.121, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2048, + "latency_ms": 38305.676, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2104, + "latency_ms": 38291.063, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2033, + "latency_ms": 36707.793, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1978, + "latency_ms": 37369.289, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 2055, + "latency_ms": 40492.763, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1553, + "latency_ms": 38292.096, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2058, + "latency_ms": 39665.382, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2006, + "latency_ms": 37113.167, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 682, + "latency_ms": 37341.622, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2015, + "latency_ms": 39492.744, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1981, + "latency_ms": 38686.353, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 2120, + "latency_ms": 39024.511, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 1787, + "latency_ms": 38353.819, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1996, + "latency_ms": 38190.288, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1949, + "latency_ms": 38190.125, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2107, + "latency_ms": 38699.416, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 2071, + "latency_ms": 38331.537, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 2039, + "latency_ms": 37517.041, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2006, + "latency_ms": 36227.883, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1985, + "latency_ms": 36826.573, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2012, + "latency_ms": 37418.965, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2028, + "latency_ms": 38542.206, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 760, + "latency_ms": 39930.082, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1906, + "latency_ms": 37201.275, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 2146, + "latency_ms": 37335.95, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2114, + "latency_ms": 37971.004, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1988, + "latency_ms": 36669.72, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 2042, + "latency_ms": 37038.282, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 2102, + "latency_ms": 39403.885, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1999, + "latency_ms": 37584.045, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 899, + "latency_ms": 37252.886, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1981, + "latency_ms": 38028.711, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2022, + "latency_ms": 37598.995, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 2053, + "latency_ms": 38532.648, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2119, + "latency_ms": 38322.881, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 2121, + "latency_ms": 37214.662, + "indexed_revision_matches": true + }, + { + "provider_id": "repowise", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 2018, + "latency_ms": 36737.637, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/ripgrep.json b/benchmarks/context-retrieval/results/full-field-got/ripgrep.json new file mode 100644 index 00000000..2ca2ee2a --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/ripgrep.json @@ -0,0 +1,3131 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "ripgrep", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "test/timeout.ts", + "source/create.ts", + "source/index.ts", + "source/types.ts", + "test/arguments.ts" + ] + }, + { + "provider_id": "ripgrep", + "query": "Fix a regression where body was sent after redirect", + "revision": "25ae938e441ee1fb35bcfa76582d1075cc5c9eef", + "returned": [ + "test/agent.ts", + "test/cancel.ts", + "test/stream.ts", + "source/core/index.ts", + "source/create.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "ripgrep", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.4073, + "full_recall_at_5": 0.1944, + "mean_recall_at_10": 0.5858, + "full_recall_at_10": 0.3704, + "mean_recall_at_20": 0.8177, + "full_recall_at_20": 0.713, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1111, + "mean_recall_at_16000_tokens": 0.3887, + "mean_precision_at_10": 0.1227, + "zero_hit_rate": 0.0926, + "median_tokens_delivered": 63496, + "median_latency_ms": 175.1625, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 68, + "mean_recall_at_5": 0.2191, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.3422, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.7105, + "full_recall_at_20": 0.5441, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0221, + "mean_recall_at_16000_tokens": 0.202, + "mean_precision_at_10": 0.0853, + "zero_hit_rate": 0.1471, + "median_tokens_delivered": 67054, + "median_latency_ms": 179.837, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 40, + "mean_recall_at_5": 0.7271, + "full_recall_at_5": 0.525, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2625, + "mean_recall_at_16000_tokens": 0.7063, + "mean_precision_at_10": 0.1863, + "zero_hit_rate": 0, + "median_tokens_delivered": 58861, + "median_latency_ms": 170.1415, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.4488, + "full_recall_at_5": 0.2222, + "mean_recall_at_10": 0.6222, + "full_recall_at_10": 0.4074, + "mean_recall_at_20": 0.8346, + "full_recall_at_20": 0.6852, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1759, + "mean_recall_at_16000_tokens": 0.4333, + "mean_precision_at_10": 0.1278, + "zero_hit_rate": 0.037, + "median_tokens_delivered": 63714.5, + "median_latency_ms": 182.1265, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3657, + "full_recall_at_5": 0.1667, + "mean_recall_at_10": 0.5494, + "full_recall_at_10": 0.3333, + "mean_recall_at_20": 0.8009, + "full_recall_at_20": 0.7407, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0463, + "mean_recall_at_16000_tokens": 0.3441, + "mean_precision_at_10": 0.1176, + "zero_hit_rate": 0.1481, + "median_tokens_delivered": 63227.5, + "median_latency_ms": 164.141, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 16955.4, + "peak_rss_mb": 20639.2, + "delta_rss_mb": 3683.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "ripgrep", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 53715, + "latency_ms": 196.628, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 75976, + "latency_ms": 165.438, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 77391, + "latency_ms": 164.747, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 62445, + "latency_ms": 143.903, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 67858, + "latency_ms": 139.031, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 155510, + "latency_ms": 172.97, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 53641, + "latency_ms": 145.713, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 47855, + "latency_ms": 136.103, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87223, + "latency_ms": 184.58, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 44508, + "latency_ms": 102.537, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 102374, + "latency_ms": 123.873, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 38716, + "latency_ms": 100.57, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 42620, + "latency_ms": 126.025, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 47089, + "latency_ms": 111.297, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 64066, + "latency_ms": 131.84, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 85558, + "latency_ms": 138.421, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 70344, + "latency_ms": 131.668, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 65587, + "latency_ms": 141.533, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 84000, + "latency_ms": 144.486, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 45399, + "latency_ms": 99.15, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 73783, + "latency_ms": 112.147, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 48328, + "latency_ms": 128.874, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 80402, + "latency_ms": 156.634, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 43123, + "latency_ms": 107.323, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 76090, + "latency_ms": 114.93, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45814, + "latency_ms": 115.904, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 61024, + "latency_ms": 120.529, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 41278, + "latency_ms": 87.25, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 64014, + "latency_ms": 103.627, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 63688, + "latency_ms": 119.718, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 46609, + "latency_ms": 78.735, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 63629, + "latency_ms": 123.202, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 38792, + "latency_ms": 101.012, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 96533, + "latency_ms": 137.134, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62246, + "latency_ms": 156.159, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 100844, + "latency_ms": 203.047, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 87406, + "latency_ms": 368.009, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 49174, + "latency_ms": 185.829, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [ + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 23196, + "latency_ms": 109.43, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 143218, + "latency_ms": 199.378, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 66079, + "latency_ms": 128.461, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 83537, + "latency_ms": 200.353, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 88311, + "latency_ms": 299.615, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 27978, + "latency_ms": 136.769, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 45694, + "latency_ms": 153.855, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53523, + "latency_ms": 154.218, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 81742, + "latency_ms": 161.284, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 86165, + "latency_ms": 197.643, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 85215, + "latency_ms": 173.699, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 63039, + "latency_ms": 148.351, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 80784, + "latency_ms": 191.283, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 48920, + "latency_ms": 381.802, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 137075, + "latency_ms": 191.3, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 64113, + "latency_ms": 129.818, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 75702, + "latency_ms": 175.03, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 61082, + "latency_ms": 184.379, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 46379, + "latency_ms": 139.627, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 41008, + "latency_ms": 143.747, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 37313, + "latency_ms": 126.33, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 66786, + "latency_ms": 409.614, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 49300, + "latency_ms": 147.123, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 52150, + "latency_ms": 163.535, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 35590, + "latency_ms": 166.494, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 74537, + "latency_ms": 176.794, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 50943, + "latency_ms": 273.323, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 83669, + "latency_ms": 234.902, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42460, + "latency_ms": 274.825, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38629, + "latency_ms": 159.385, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 57707, + "latency_ms": 179.874, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 61748, + "latency_ms": 192.907, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 70220, + "latency_ms": 293.153, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 77408, + "latency_ms": 201.047, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 75447, + "latency_ms": 248.259, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 67885, + "latency_ms": 383.557, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 31610, + "latency_ms": 173.789, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 145671, + "latency_ms": 236.756, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48539, + "latency_ms": 264.702, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 154510, + "latency_ms": 238.682, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 54485, + "latency_ms": 176.853, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27502, + "latency_ms": 122.12, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 87615, + "latency_ms": 175.295, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40259, + "latency_ms": 238.902, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 60015, + "latency_ms": 232.99, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 53717, + "latency_ms": 371.687, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 44547, + "latency_ms": 251.448, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 69438, + "latency_ms": 269.864, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 76447, + "latency_ms": 235.583, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 76806, + "latency_ms": 227.979, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 61652, + "latency_ms": 240.171, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 48300, + "latency_ms": 168.511, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 47055, + "latency_ms": 244.569, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 79869, + "latency_ms": 451.314, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 79302, + "latency_ms": 252.542, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33797, + "latency_ms": 86.792, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 50056, + "latency_ms": 230.379, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 91929, + "latency_ms": 323.059, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 62826, + "latency_ms": 335.452, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 35876, + "latency_ms": 260.76, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 82575, + "latency_ms": 518.181, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 83865, + "latency_ms": 400.382, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 68966, + "latency_ms": 243.432, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37763, + "latency_ms": 198.03, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 90708, + "latency_ms": 284.427, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 55362, + "latency_ms": 241.099, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 63363, + "latency_ms": 197.286, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 85726, + "latency_ms": 267.128, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 86911, + "latency_ms": 481.827, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "test/promise.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 67322, + "latency_ms": 245.213, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/semble.json b/benchmarks/context-retrieval/results/full-field-got/semble.json new file mode 100644 index 00000000..707b917b --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/semble.json @@ -0,0 +1,3131 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "semble", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "source/core/index.ts", + "source/core/utils/dns-ip-version.ts", + "source/index.ts", + "source/create.ts", + "benchmark/index.ts" + ] + }, + { + "provider_id": "semble", + "query": "Fix `Invalid URL` checks on Node.js 16", + "revision": "a06e7585ab4503e1dd7178d83dd4443ff6dda6c1", + "returned": [ + "source/core/options.ts", + "benchmark/index.ts", + "source/core/index.ts", + "source/core/errors.ts", + "source/create.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.5755, + "full_recall_at_5": 0.2315, + "mean_recall_at_10": 0.7452, + "full_recall_at_10": 0.5463, + "mean_recall_at_20": 0.8495, + "full_recall_at_20": 0.7407, + "mean_recall_at_1000_tokens": 0.4782, + "mean_recall_at_4000_tokens": 0.8357, + "mean_recall_at_16000_tokens": 0.8495, + "mean_precision_at_10": 0.1454, + "zero_hit_rate": 0.0648, + "median_tokens_delivered": 3939, + "median_latency_ms": 1787.8075, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 49, + "mean_recall_at_5": 0.4214, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.4384, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.6684, + "full_recall_at_20": 0.4286, + "mean_recall_at_1000_tokens": 0.3534, + "mean_recall_at_4000_tokens": 0.6378, + "mean_recall_at_16000_tokens": 0.6684, + "mean_precision_at_10": 0.1122, + "zero_hit_rate": 0.1429, + "median_tokens_delivered": 3989, + "median_latency_ms": 1801.442, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 59, + "mean_recall_at_5": 0.7034, + "full_recall_at_5": 0.4237, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.5819, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1729, + "zero_hit_rate": 0, + "median_tokens_delivered": 3890, + "median_latency_ms": 1773.745, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.5784, + "full_recall_at_5": 0.2222, + "mean_recall_at_10": 0.7698, + "full_recall_at_10": 0.5926, + "mean_recall_at_20": 0.8673, + "full_recall_at_20": 0.7778, + "mean_recall_at_1000_tokens": 0.4765, + "mean_recall_at_4000_tokens": 0.8395, + "mean_recall_at_16000_tokens": 0.8673, + "mean_precision_at_10": 0.1556, + "zero_hit_rate": 0.0556, + "median_tokens_delivered": 3958.5, + "median_latency_ms": 1836.071, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.5725, + "full_recall_at_5": 0.2407, + "mean_recall_at_10": 0.7207, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.8318, + "full_recall_at_20": 0.7037, + "mean_recall_at_1000_tokens": 0.4799, + "mean_recall_at_4000_tokens": 0.8318, + "mean_recall_at_16000_tokens": 0.8318, + "mean_precision_at_10": 0.1352, + "zero_hit_rate": 0.0741, + "median_tokens_delivered": 3926, + "median_latency_ms": 1676.2565, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 22746.6, + "peak_rss_mb": 37976, + "delta_rss_mb": 15229.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "semble", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3889, + "latency_ms": 1352.594, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3983, + "latency_ms": 1416.523, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3730, + "latency_ms": 1249.783, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 4085, + "latency_ms": 1249.701, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3990, + "latency_ms": 1601.703, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/http.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4053, + "latency_ms": 1877.738, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4247, + "latency_ms": 1631.244, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4063, + "latency_ms": 1964.533, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3912, + "latency_ms": 1814.476, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3664, + "latency_ms": 1529.528, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3786, + "latency_ms": 2588.572, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3480, + "latency_ms": 1987.574, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/normalize-arguments.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 3961, + "latency_ms": 1307.173, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3884, + "latency_ms": 1301.215, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3809, + "latency_ms": 3398.417, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4921, + "latency_ms": 2481.36, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4037, + "latency_ms": 1608.117, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3956, + "latency_ms": 1373.338, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4208, + "latency_ms": 1844.94, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3761, + "latency_ms": 1842.118, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3490, + "latency_ms": 1612.152, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4119, + "latency_ms": 1305.182, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3737, + "latency_ms": 1996.812, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3516, + "latency_ms": 2043.048, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4334, + "latency_ms": 1627.768, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3709, + "latency_ms": 1477.969, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3866, + "latency_ms": 1551.177, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4272, + "latency_ms": 1980.339, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3946, + "latency_ms": 1371.723, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3819, + "latency_ms": 1043.972, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.75, + "precision_at_5": 0.6, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 4152, + "latency_ms": 1462.492, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4925, + "latency_ms": 2001.256, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 3928, + "latency_ms": 1803.389, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3878, + "latency_ms": 1806.176, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4437, + "latency_ms": 1325.094, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4575, + "latency_ms": 1638.176, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4277, + "latency_ms": 2452.52, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3938, + "latency_ms": 1603.898, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 3482, + "latency_ms": 1241.113, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3775, + "latency_ms": 1772.29, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3900, + "latency_ms": 1762.943, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4630, + "latency_ms": 2078.283, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4057, + "latency_ms": 1698.031, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3672, + "latency_ms": 1469.194, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2143, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3731, + "latency_ms": 1360.152, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3967, + "latency_ms": 2053.421, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3768, + "latency_ms": 1579.102, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4119, + "latency_ms": 1789.868, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4505, + "latency_ms": 5017.076, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts" + ], + "missed": [ + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.6, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 4, + "tokens_delivered": 4048, + "latency_ms": 1834.907, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3746, + "latency_ms": 1576.606, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 3940, + "latency_ms": 1490.988, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3981, + "latency_ms": 1837.235, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3506, + "latency_ms": 1360.212, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 4344, + "latency_ms": 1801.442, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3577, + "latency_ms": 1209.589, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4030, + "latency_ms": 1138.311, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3840, + "latency_ms": 1749.249, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3828, + "latency_ms": 1539.374, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3648, + "latency_ms": 1434.724, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3409, + "latency_ms": 1166.418, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 4164, + "latency_ms": 1728.119, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3938, + "latency_ms": 1697.058, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3643, + "latency_ms": 1797.321, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3925, + "latency_ms": 1401.99, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4073, + "latency_ms": 1604.146, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3567, + "latency_ms": 1629.828, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3832, + "latency_ms": 1860.492, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3655, + "latency_ms": 1393.75, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3888, + "latency_ms": 1654.482, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4102, + "latency_ms": 1545.359, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3702, + "latency_ms": 1644.613, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3880, + "latency_ms": 1494.975, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.1875, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4377, + "latency_ms": 1498.324, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3979, + "latency_ms": 1707.638, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2308, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3847, + "latency_ms": 2522.944, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3793, + "latency_ms": 1327.144, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4094, + "latency_ms": 2294.423, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4403, + "latency_ms": 1976.359, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3494, + "latency_ms": 1773.745, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4615, + "latency_ms": 1874.613, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4071, + "latency_ms": 3857.511, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3531, + "latency_ms": 1915.794, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4418, + "latency_ms": 1874.373, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4151, + "latency_ms": 1916.482, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4262, + "latency_ms": 2004.839, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3781, + "latency_ms": 1624.667, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3937, + "latency_ms": 6319.692, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 3989, + "latency_ms": 5020.064, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3803, + "latency_ms": 4206.52, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4047, + "latency_ms": 3837.478, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4768, + "latency_ms": 2326.036, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4361, + "latency_ms": 1985.913, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4241, + "latency_ms": 2766.904, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3890, + "latency_ms": 1848.103, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 4162, + "latency_ms": 2116.904, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3957, + "latency_ms": 1989.103, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 3745, + "latency_ms": 1785.747, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 4249, + "latency_ms": 1895.395, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3841, + "latency_ms": 3570.16, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3516, + "latency_ms": 8209.465, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3855, + "latency_ms": 9748.75, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 4196, + "latency_ms": 7711.335, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3714, + "latency_ms": 7478.121, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "source/create.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 4026, + "latency_ms": 5562.714, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/extend.types.ts" + ], + "missed": [ + "test/create.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.4, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.2, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 1, + "tokens_delivered": 3612, + "latency_ms": 4463.594, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4150, + "latency_ms": 4246.246, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 4113, + "latency_ms": 3818.707, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/token-savior-regex.json b/benchmarks/context-retrieval/results/full-field-got/token-savior-regex.json new file mode 100644 index 00000000..c891cff5 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/token-savior-regex.json @@ -0,0 +1,3152 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "token-savior-regex", + "query": "Fix aborting during download progress", + "revision": "e9489c1729a56711a18aa49d11c1ed596dfef451", + "returned": [ + "documentation/1-promise.md", + "readme.md", + "source/core/options.ts", + "test/abort.ts", + "documentation/3-streams.md" + ] + }, + { + "provider_id": "token-savior-regex", + "query": "Fix unhandled `The server aborted pending request` rejection", + "revision": "04f3ea4b7995e1609555dd13f00641b8f470de4c", + "returned": [ + "readme.md", + "source/as-promise/index.ts", + "source/core/index.ts", + "test/cancel.ts", + "test/http.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "token-savior-regex", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.1539, + "full_recall_at_5": 0.0741, + "mean_recall_at_10": 0.3147, + "full_recall_at_10": 0.1574, + "mean_recall_at_20": 0.3995, + "full_recall_at_20": 0.213, + "mean_recall_at_1000_tokens": 0.1881, + "mean_recall_at_4000_tokens": 0.3949, + "mean_recall_at_16000_tokens": 0.3995, + "mean_precision_at_10": 0.0652, + "zero_hit_rate": 0.3981, + "median_tokens_delivered": 2274, + "median_latency_ms": 2771.4205, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 91, + "mean_recall_at_5": 0.0837, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1866, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.2874, + "full_recall_at_20": 0.0659, + "mean_recall_at_1000_tokens": 0.0987, + "mean_recall_at_4000_tokens": 0.2819, + "mean_recall_at_16000_tokens": 0.2874, + "mean_precision_at_10": 0.0487, + "zero_hit_rate": 0.4725, + "median_tokens_delivered": 2298, + "median_latency_ms": 2791.042, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 17, + "mean_recall_at_5": 0.5294, + "full_recall_at_5": 0.4706, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0.6667, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.1536, + "zero_hit_rate": 0, + "median_tokens_delivered": 2048, + "median_latency_ms": 2691.579, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.1395, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.279, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.3809, + "full_recall_at_20": 0.1852, + "mean_recall_at_1000_tokens": 0.1988, + "mean_recall_at_4000_tokens": 0.3809, + "mean_recall_at_16000_tokens": 0.3809, + "mean_precision_at_10": 0.0572, + "zero_hit_rate": 0.4074, + "median_tokens_delivered": 2355.5, + "median_latency_ms": 3090.9125, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.1682, + "full_recall_at_5": 0.1111, + "mean_recall_at_10": 0.3503, + "full_recall_at_10": 0.2037, + "mean_recall_at_20": 0.4182, + "full_recall_at_20": 0.2407, + "mean_recall_at_1000_tokens": 0.1775, + "mean_recall_at_4000_tokens": 0.409, + "mean_recall_at_16000_tokens": 0.4182, + "mean_precision_at_10": 0.0733, + "zero_hit_rate": 0.3889, + "median_tokens_delivered": 2239, + "median_latency_ms": 2469.8495, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 108 + }, + "resources": { + "baseline_rss_mb": 18504.5, + "peak_rss_mb": 23255.6, + "delta_rss_mb": 4751.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "token-savior-regex", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1676, + "latency_ms": 1946.636, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2575, + "latency_ms": 1785.217, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2284, + "latency_ms": 2013.565, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1430, + "latency_ms": 1477.878, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3241, + "latency_ms": 2201.076, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4989, + "latency_ms": 3897.996, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 2231, + "latency_ms": 3596.84, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 870, + "latency_ms": 1760.273, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1367, + "latency_ms": 4222.562, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1969, + "latency_ms": 1948.867, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2035, + "latency_ms": 2259.572, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1465, + "latency_ms": 1634.446, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1693, + "latency_ms": 1951.038, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1407, + "latency_ms": 1583.571, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 3800, + "latency_ms": 3818.32, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2525, + "latency_ms": 4044.86, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 3434, + "latency_ms": 4074.878, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 2672, + "latency_ms": 4058.654, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 19, + "tokens_delivered": 2413, + "latency_ms": 4084.919, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2568, + "latency_ms": 1986.036, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 2929, + "latency_ms": 2612.689, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2701, + "latency_ms": 3306.294, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 3217, + "latency_ms": 4047.826, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1159, + "latency_ms": 1429.154, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2170, + "latency_ms": 1792.948, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 1692, + "latency_ms": 2233.507, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 1656, + "latency_ms": 3468.394, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 926, + "latency_ms": 3827.57, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 2358, + "latency_ms": 4781.82, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 2253, + "latency_ms": 1849.842, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.3333, + "recall_at_10": 0.25, + "precision_at_10": 0.3333, + "recall_at_20": 0.25, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 518, + "latency_ms": 2023.951, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1470, + "latency_ms": 2341.257, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2073, + "latency_ms": 2342.338, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 13, + "tokens_delivered": 4109, + "latency_ms": 3946.79, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1566, + "latency_ms": 1104.884, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 5, + "tokens_delivered": 4031, + "latency_ms": 3860.086, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2640, + "latency_ms": 3225.66, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 2477, + "latency_ms": 3244.433, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 810, + "latency_ms": 2087.501, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 2671, + "latency_ms": 3728.143, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1659, + "latency_ms": 1691.633, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 16, + "tokens_delivered": 5111, + "latency_ms": 4144.086, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1582, + "latency_ms": 2280.216, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 739, + "latency_ms": 1480.472, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 8, + "tokens_delivered": 2621, + "latency_ms": 3303.567, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1216, + "latency_ms": 1954.197, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3046, + "latency_ms": 2918.491, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 12, + "tokens_delivered": 2765, + "latency_ms": 4349.888, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 14, + "tokens_delivered": 2957, + "latency_ms": 3360.921, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.4, + "recall_at_4000_tokens": 0.4, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 5, + "tokens_delivered": 1489, + "latency_ms": 2002.2, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 3505, + "latency_ms": 2806.167, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2825, + "latency_ms": 3924.581, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 20, + "tokens_delivered": 2759, + "latency_ms": 3158.607, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3046, + "latency_ms": 2284.422, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2463, + "latency_ms": 4522.424, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 2040, + "latency_ms": 2349.608, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2586, + "latency_ms": 3147.319, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1355, + "latency_ms": 2748.211, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 828, + "latency_ms": 1891.54, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 7, + "tokens_delivered": 2624, + "latency_ms": 2751.799, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2048, + "latency_ms": 2329.507, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 8, + "tokens_delivered": 2797, + "latency_ms": 2354.32, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1800, + "latency_ms": 2617.469, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 3174, + "latency_ms": 2691.579, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2822, + "latency_ms": 3192.29, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1563, + "latency_ms": 2791.042, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 745, + "latency_ms": 1629.38, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1562, + "latency_ms": 1284.382, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2207, + "latency_ms": 3064.41, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 1621, + "latency_ms": 2585.379, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 3248, + "latency_ms": 3117.415, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 3307, + "latency_ms": 6258.801, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3565, + "latency_ms": 2703.364, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6667, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 11, + "tokens_delivered": 2264, + "latency_ms": 3052.27, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1613, + "latency_ms": 1991.188, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 9, + "tokens_delivered": 5171, + "latency_ms": 4620.303, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1942, + "latency_ms": 1905.055, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2420, + "latency_ms": 2920.457, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1669, + "latency_ms": 1938.018, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 651, + "latency_ms": 1288.003, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2298, + "latency_ms": 2862.713, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 807, + "latency_ms": 2711.551, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 2186, + "latency_ms": 2096.904, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 1791, + "latency_ms": 4280.958, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1404, + "latency_ms": 3298.953, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 3291, + "latency_ms": 2950.241, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 5242, + "latency_ms": 4645.209, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3357, + "latency_ms": 4167.691, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.25, + "recall_at_20": 0.6667, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 5, + "tokens_delivered": 2295, + "latency_ms": 2067.896, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2225, + "latency_ms": 3158.823, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3290, + "latency_ms": 3415.006, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 4365, + "latency_ms": 4116.388, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5705, + "latency_ms": 4382.778, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 343, + "latency_ms": 1293.153, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 926, + "latency_ms": 1821.615, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6401, + "latency_ms": 4266.004, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1076, + "latency_ms": 2964.851, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1682, + "latency_ms": 3595.73, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 6, + "tokens_delivered": 2725, + "latency_ms": 4055.566, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5396, + "latency_ms": 4258.598, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 1871, + "latency_ms": 1991.696, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 912, + "latency_ms": 1667.658, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 19, + "tokens_delivered": 2992, + "latency_ms": 2686.085, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1577, + "latency_ms": 1362.996, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 1632, + "latency_ms": 2225.891, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3966, + "latency_ms": 3478.79, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 3334, + "latency_ms": 3214.819, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior-regex", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3919, + "latency_ms": 3747.97, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/full-field-got/token-savior.json b/benchmarks/context-retrieval/results/full-field-got/token-savior.json new file mode 100644 index 00000000..532bac29 --- /dev/null +++ b/benchmarks/context-retrieval/results/full-field-got/token-savior.json @@ -0,0 +1,3119 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "tier": "small", + "tier_code_files": 85, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": false, + "missing": [ + "random-files", + "random-code-files", + "churn-ranked" + ], + "reason": "controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric." + }, + "controls_lose": { + "ok": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "token-savior", + "query": "Fix `dnsCache: true` having no effect", + "revision": "e02845f1aa737d55dce23381d0f4f2a61b1eb5e1", + "returned": [ + "source/core/utils/dns-ip-version.ts", + "source/core/index.ts" + ] + }, + { + "provider_id": "token-savior", + "query": "Fix abort event listeners not always being cleaned up (#2162)", + "revision": "5f278d74125608b7abe75941cb6a71e21e0fb892", + "returned": [ + "test/abort.ts" + ] + } + ], + "trustworthy": false + }, + "providers": [ + { + "provider_id": "token-savior", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.0864, + "full_recall_at_5": 0.0185, + "mean_recall_at_10": 0.0864, + "full_recall_at_10": 0.0185, + "mean_recall_at_20": 0.0864, + "full_recall_at_20": 0.0185, + "mean_recall_at_1000_tokens": 0.0864, + "mean_recall_at_4000_tokens": 0.0864, + "mean_recall_at_16000_tokens": 0.0864, + "mean_precision_at_10": 0.1466, + "zero_hit_rate": 0.8426, + "median_tokens_delivered": 24, + "median_latency_ms": 20969.6465, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 106, + "mean_recall_at_5": 0.0692, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0692, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0692, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.0692, + "mean_recall_at_4000_tokens": 0.0692, + "mean_recall_at_16000_tokens": 0.0692, + "mean_precision_at_10": 0.1368, + "zero_hit_rate": 0.8585, + "median_tokens_delivered": 24, + "median_latency_ms": 20836.374, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 1, + "full_recall_at_5": 1, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 1, + "mean_recall_at_4000_tokens": 1, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.6667, + "zero_hit_rate": 0, + "median_tokens_delivered": 110.5, + "median_latency_ms": 40664.177, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.1204, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.1204, + "full_recall_at_10": 0.037, + "mean_recall_at_20": 0.1204, + "full_recall_at_20": 0.037, + "mean_recall_at_1000_tokens": 0.1204, + "mean_recall_at_4000_tokens": 0.1204, + "mean_recall_at_16000_tokens": 0.1204, + "mean_precision_at_10": 0.1914, + "zero_hit_rate": 0.7963, + "median_tokens_delivered": 24, + "median_latency_ms": 20090.9895, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.0525, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0525, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0525, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.0525, + "mean_recall_at_4000_tokens": 0.0525, + "mean_recall_at_16000_tokens": 0.0525, + "mean_precision_at_10": 0.1019, + "zero_hit_rate": 0.8889, + "median_tokens_delivered": 24, + "median_latency_ms": 22591.6335, + "payload_kind": [ + "tool-output" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 27, + "indexed-but-returned-nothing": 81 + }, + "resources": { + "baseline_rss_mb": 18483.1, + "peak_rss_mb": 43238.9, + "delta_rss_mb": 24755.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "RELIABILITY GATE FAILED — do not publish these numbers: controls absent: random-files, random-code-files, churn-ranked. A query-blind control is the only thing that detects a broken metric.", + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "token-savior", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 265, + "latency_ms": 18417.345, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 338, + "latency_ms": 35909.441, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67, + "latency_ms": 42676.991, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 256, + "latency_ms": 30433.324, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 71, + "latency_ms": 31953.748, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 155, + "latency_ms": 64881.15, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 154, + "latency_ms": 44520.147, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 109, + "latency_ms": 36018.209, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 65539.241, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 113, + "latency_ms": 43830.049, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 66, + "latency_ms": 144510.51, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 95608.855, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 203, + "latency_ms": 35465.641, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 67, + "latency_ms": 36808.207, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 61, + "latency_ms": 46102.197, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 68, + "latency_ms": 45336.838, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 62, + "latency_ms": 44917.904, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 71, + "latency_ms": 67715.512, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 80605.038, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 85981.451, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 103872.971, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 29881.089, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 46135.75, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 47815.095, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 60680.512, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [], + "missed": [ + "package.json", + "tsconfig.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 51852.493, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 66, + "latency_ms": 52086.789, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 59304.518, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 75899.71, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 45681.015, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 56894.293, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 64, + "latency_ms": 25536.446, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 13950.1, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 40845.076, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 19118.701, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 41915.242, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 36879.757, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 80, + "latency_ms": 17212.074, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 12869.675, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 69, + "latency_ms": 53894.322, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 21349.249, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 29450.474, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 109, + "latency_ms": 32950.651, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 20341.886, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 14393.295, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 30943.174, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 30403.058, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 19692.271, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 25217.749, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 19213.628, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 23341.718, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 12363.556, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 36309.411, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73, + "latency_ms": 18043.11, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 21801.233, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 11727.061, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 10188.711, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 23042.619, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 10259.663, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 12049.242, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 12249.651, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 13548.82, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 77, + "latency_ms": 12089.339, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 20970.942, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 10813.18, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 67, + "latency_ms": 29293.179, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 17236.71, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 16486.448, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 11851.309, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 18679.288, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 14685.971, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 15288.451, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 17075.789, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 15095.201, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 9926.836, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 1, + "recall_at_10": 0.6667, + "precision_at_10": 1, + "recall_at_20": 0.6667, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 185, + "latency_ms": 28639.074, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 17491.681, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 88, + "latency_ms": 40533.66, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 16487.477, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 15764.905, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 21245.125, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 18606.078, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 15706.382, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 16573.485, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 10461.599, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 16204.805, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 16027.009, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 16962.325, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 20580.766, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 13943.297, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 13921.765, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 22938.45, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 20968.351, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 24398.5, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 10849.064, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 15254.108, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 20704.397, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 9508.903, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 16048.57, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 16033.61, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 13643.721, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 15623.52, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 21841.549, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 15352.237, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 15508.108, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 23686.869, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 16022.578, + "indexed_revision_matches": true + }, + { + "provider_id": "token-savior", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24, + "latency_ms": 10384.767, + "indexed_revision_matches": true + } + ] +} diff --git a/benchmarks/context-retrieval/results/score-pub-express.json b/benchmarks/context-retrieval/results/score-pub-express.json new file mode 100644 index 00000000..8ff2f3a4 --- /dev/null +++ b/benchmarks/context-retrieval/results/score-pub-express.json @@ -0,0 +1,5142 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "express", + "head": "a3714473feb3d2908add734d340e7755fd85e0a3" + }, + "corpus_counts": { + "cases": 28, + "path_leak": 9, + "baseline_blind": 19, + "multi_file": 14 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "providers": [ + { + "provider_id": "agent-default", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.6369, + "full_recall_at_5": 0.5357, + "mean_recall_at_10": 0.6786, + "full_recall_at_10": 0.6429, + "mean_recall_at_20": 0.7321, + "full_recall_at_20": 0.6786, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.244, + "mean_recall_at_16000_tokens": 0.6488, + "mean_precision_at_10": 0.1107, + "zero_hit_rate": 0.2143, + "median_tokens_delivered": 41515.5, + "median_latency_ms": 120.873, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 10, + "mean_recall_at_5": 0.1, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.25, + "full_recall_at_20": 0.1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.05, + "mean_recall_at_16000_tokens": 0.1, + "mean_precision_at_10": 0.02, + "zero_hit_rate": 0.6, + "median_tokens_delivered": 38930, + "median_latency_ms": 132.381, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 18, + "mean_recall_at_5": 0.9352, + "full_recall_at_5": 0.8333, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.3519, + "mean_recall_at_16000_tokens": 0.9537, + "mean_precision_at_10": 0.1611, + "zero_hit_rate": 0, + "median_tokens_delivered": 41515.5, + "median_latency_ms": 120.873, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.8519, + "full_recall_at_5": 0.7778, + "mean_recall_at_10": 0.8889, + "full_recall_at_10": 0.8889, + "mean_recall_at_20": 0.8889, + "full_recall_at_20": 0.8889, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.3704, + "mean_recall_at_16000_tokens": 0.8889, + "mean_precision_at_10": 0.1667, + "zero_hit_rate": 0.1111, + "median_tokens_delivered": 40896, + "median_latency_ms": 131.911, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.5351, + "full_recall_at_5": 0.4211, + "mean_recall_at_10": 0.5789, + "full_recall_at_10": 0.5263, + "mean_recall_at_20": 0.6579, + "full_recall_at_20": 0.5789, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1842, + "mean_recall_at_16000_tokens": 0.5351, + "mean_precision_at_10": 0.0842, + "zero_hit_rate": 0.2632, + "median_tokens_delivered": 43299, + "median_latency_ms": 114.895, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 76488.7, + "peak_rss_mb": 76496.1, + "delta_rss_mb": 7.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "keyword-search", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.5714, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.6696, + "full_recall_at_10": 0.6071, + "mean_recall_at_20": 0.7143, + "full_recall_at_20": 0.6429, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.4405, + "mean_recall_at_16000_tokens": 0.5714, + "mean_precision_at_10": 0.1143, + "zero_hit_rate": 0.2143, + "median_tokens_delivered": 44093.5, + "median_latency_ms": 84.5815, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 10, + "mean_recall_at_5": 0.1, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.225, + "full_recall_at_10": 0.1, + "mean_recall_at_20": 0.25, + "full_recall_at_20": 0.1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1, + "mean_recall_at_16000_tokens": 0.1, + "mean_precision_at_10": 0.05, + "zero_hit_rate": 0.6, + "median_tokens_delivered": 43029, + "median_latency_ms": 78.7575, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 18, + "mean_recall_at_5": 0.8333, + "full_recall_at_5": 0.7778, + "mean_recall_at_10": 0.9167, + "full_recall_at_10": 0.8889, + "mean_recall_at_20": 0.9722, + "full_recall_at_20": 0.9444, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.6296, + "mean_recall_at_16000_tokens": 0.8333, + "mean_precision_at_10": 0.15, + "zero_hit_rate": 0, + "median_tokens_delivered": 45008.5, + "median_latency_ms": 84.5815, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.5556, + "full_recall_at_5": 0.4444, + "mean_recall_at_10": 0.7222, + "full_recall_at_10": 0.6667, + "mean_recall_at_20": 0.8333, + "full_recall_at_20": 0.7778, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2778, + "mean_recall_at_16000_tokens": 0.5556, + "mean_precision_at_10": 0.1444, + "zero_hit_rate": 0.1111, + "median_tokens_delivered": 44888, + "median_latency_ms": 92.347, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.5789, + "full_recall_at_5": 0.5263, + "mean_recall_at_10": 0.6447, + "full_recall_at_10": 0.5789, + "mean_recall_at_20": 0.6579, + "full_recall_at_20": 0.5789, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.5175, + "mean_recall_at_16000_tokens": 0.5789, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.2632, + "median_tokens_delivered": 43299, + "median_latency_ms": 72.416, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 76506.8, + "peak_rss_mb": 76649.7, + "delta_rss_mb": 142.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codesearch", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.5714, + "full_recall_at_5": 0.5357, + "mean_recall_at_10": 0.5893, + "full_recall_at_10": 0.5714, + "mean_recall_at_20": 0.5893, + "full_recall_at_20": 0.5714, + "mean_recall_at_1000_tokens": 0.4286, + "mean_recall_at_4000_tokens": 0.5893, + "mean_recall_at_16000_tokens": 0.5893, + "mean_precision_at_10": 0.1845, + "zero_hit_rate": 0.3929, + "median_tokens_delivered": 1423, + "median_latency_ms": 2118.707, + "payload_kind": [ + "chunks" + ], + "unavailable": 2 + }, + "baseline_missed": { + "cases": 10, + "mean_recall_at_5": 0.3, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.35, + "full_recall_at_10": 0.3, + "mean_recall_at_20": 0.35, + "full_recall_at_20": 0.3, + "mean_recall_at_1000_tokens": 0.3, + "mean_recall_at_4000_tokens": 0.35, + "mean_recall_at_16000_tokens": 0.35, + "mean_precision_at_10": 0.0686, + "zero_hit_rate": 0.6, + "median_tokens_delivered": 1331, + "median_latency_ms": 2246.733, + "payload_kind": [ + "chunks" + ], + "unavailable": 1 + }, + "baseline_solved": { + "cases": 18, + "mean_recall_at_5": 0.7222, + "full_recall_at_5": 0.7222, + "mean_recall_at_10": 0.7222, + "full_recall_at_10": 0.7222, + "mean_recall_at_20": 0.7222, + "full_recall_at_20": 0.7222, + "mean_recall_at_1000_tokens": 0.5, + "mean_recall_at_4000_tokens": 0.7222, + "mean_recall_at_16000_tokens": 0.7222, + "mean_precision_at_10": 0.2489, + "zero_hit_rate": 0.2778, + "median_tokens_delivered": 1435, + "median_latency_ms": 2118.707, + "payload_kind": [ + "chunks" + ], + "unavailable": 1 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.8889, + "full_recall_at_5": 0.8889, + "mean_recall_at_10": 0.8889, + "full_recall_at_10": 0.8889, + "mean_recall_at_20": 0.8889, + "full_recall_at_20": 0.8889, + "mean_recall_at_1000_tokens": 0.8333, + "mean_recall_at_4000_tokens": 0.8889, + "mean_recall_at_16000_tokens": 0.8889, + "mean_precision_at_10": 0.3025, + "zero_hit_rate": 0.1111, + "median_tokens_delivered": 1342, + "median_latency_ms": 1897.873, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.4211, + "full_recall_at_5": 0.3684, + "mean_recall_at_10": 0.4474, + "full_recall_at_10": 0.4211, + "mean_recall_at_20": 0.4474, + "full_recall_at_20": 0.4211, + "mean_recall_at_1000_tokens": 0.2368, + "mean_recall_at_4000_tokens": 0.4474, + "mean_recall_at_16000_tokens": 0.4474, + "mean_precision_at_10": 0.1286, + "zero_hit_rate": 0.5263, + "median_tokens_delivered": 1472, + "median_latency_ms": 2586.826, + "payload_kind": [ + "chunks" + ], + "unavailable": 2 + } + }, + "resources": { + "baseline_rss_mb": 76565.8, + "peak_rss_mb": 84733.9, + "delta_rss_mb": 8168.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.5149, + "full_recall_at_5": 0.3929, + "mean_recall_at_10": 0.6042, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.6399, + "full_recall_at_20": 0.5357, + "mean_recall_at_1000_tokens": 0.5089, + "mean_recall_at_4000_tokens": 0.5685, + "mean_recall_at_16000_tokens": 0.6399, + "mean_precision_at_10": 0.1015, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 1680, + "median_latency_ms": 1331.9745, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 10, + "mean_recall_at_5": 0.125, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.275, + "full_recall_at_10": 0.2, + "mean_recall_at_20": 0.325, + "full_recall_at_20": 0.2, + "mean_recall_at_1000_tokens": 0.075, + "mean_recall_at_4000_tokens": 0.175, + "mean_recall_at_16000_tokens": 0.325, + "mean_precision_at_10": 0.0567, + "zero_hit_rate": 0.6, + "median_tokens_delivered": 1991, + "median_latency_ms": 1336.085, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 18, + "mean_recall_at_5": 0.7315, + "full_recall_at_5": 0.6111, + "mean_recall_at_10": 0.787, + "full_recall_at_10": 0.6667, + "mean_recall_at_20": 0.8148, + "full_recall_at_20": 0.7222, + "mean_recall_at_1000_tokens": 0.75, + "mean_recall_at_4000_tokens": 0.787, + "mean_recall_at_16000_tokens": 0.8148, + "mean_precision_at_10": 0.1265, + "zero_hit_rate": 0.1111, + "median_tokens_delivered": 1680, + "median_latency_ms": 1327.281, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.6667, + "full_recall_at_5": 0.5556, + "mean_recall_at_10": 0.6667, + "full_recall_at_10": 0.5556, + "mean_recall_at_20": 0.6667, + "full_recall_at_20": 0.5556, + "mean_recall_at_1000_tokens": 0.6667, + "mean_recall_at_4000_tokens": 0.6667, + "mean_recall_at_16000_tokens": 0.6667, + "mean_precision_at_10": 0.1159, + "zero_hit_rate": 0.2222, + "median_tokens_delivered": 1675, + "median_latency_ms": 1325.244, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.443, + "full_recall_at_5": 0.3158, + "mean_recall_at_10": 0.5746, + "full_recall_at_10": 0.4737, + "mean_recall_at_20": 0.6272, + "full_recall_at_20": 0.5263, + "mean_recall_at_1000_tokens": 0.4342, + "mean_recall_at_4000_tokens": 0.5219, + "mean_recall_at_16000_tokens": 0.6272, + "mean_precision_at_10": 0.0947, + "zero_hit_rate": 0.3158, + "median_tokens_delivered": 1685, + "median_latency_ms": 1334.631, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 78250.2, + "peak_rss_mb": 80885.7, + "delta_rss_mb": 2635.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 9973760 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.4345, + "full_recall_at_5": 0.2857, + "mean_recall_at_10": 0.5625, + "full_recall_at_10": 0.3929, + "mean_recall_at_20": 0.619, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2143, + "mean_recall_at_16000_tokens": 0.4881, + "mean_precision_at_10": 0.0857, + "zero_hit_rate": 0.25, + "median_tokens_delivered": 55061.5, + "median_latency_ms": 41.3195, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 10, + "mean_recall_at_5": 0.25, + "full_recall_at_5": 0.1, + "mean_recall_at_10": 0.475, + "full_recall_at_10": 0.3, + "mean_recall_at_20": 0.55, + "full_recall_at_20": 0.4, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1, + "mean_recall_at_16000_tokens": 0.4, + "mean_precision_at_10": 0.08, + "zero_hit_rate": 0.3, + "median_tokens_delivered": 60365, + "median_latency_ms": 41.25, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 18, + "mean_recall_at_5": 0.537, + "full_recall_at_5": 0.3889, + "mean_recall_at_10": 0.6111, + "full_recall_at_10": 0.4444, + "mean_recall_at_20": 0.6574, + "full_recall_at_20": 0.5556, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2778, + "mean_recall_at_16000_tokens": 0.537, + "mean_precision_at_10": 0.0889, + "zero_hit_rate": 0.2222, + "median_tokens_delivered": 53163.5, + "median_latency_ms": 41.3195, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0.3148, + "full_recall_at_5": 0.1111, + "mean_recall_at_10": 0.3148, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.3148, + "full_recall_at_20": 0.1111, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.3148, + "mean_precision_at_10": 0.0556, + "zero_hit_rate": 0.4444, + "median_tokens_delivered": 54998, + "median_latency_ms": 41.519, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.4912, + "full_recall_at_5": 0.3684, + "mean_recall_at_10": 0.6798, + "full_recall_at_10": 0.5263, + "mean_recall_at_20": 0.7632, + "full_recall_at_20": 0.6842, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.3158, + "mean_recall_at_16000_tokens": 0.5702, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.1579, + "median_tokens_delivered": 55125, + "median_latency_ms": 41.12, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 80167.1, + "peak_rss_mb": 80244.6, + "delta_rss_mb": 77.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 28, + "mean_recall_at_5": 0.0179, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0357, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0804, + "full_recall_at_20": 0.0357, + "mean_recall_at_1000_tokens": 0.0179, + "mean_recall_at_4000_tokens": 0.0179, + "mean_recall_at_16000_tokens": 0.0714, + "mean_precision_at_10": 0.0071, + "zero_hit_rate": 0.8571, + "median_tokens_delivered": 17557.5, + "median_latency_ms": 22.518, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 10, + "mean_recall_at_5": 0.05, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.05, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.075, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.05, + "mean_recall_at_4000_tokens": 0.05, + "mean_recall_at_16000_tokens": 0.05, + "mean_precision_at_10": 0.01, + "zero_hit_rate": 0.8, + "median_tokens_delivered": 24136, + "median_latency_ms": 20.494, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 18, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0278, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.0833, + "full_recall_at_20": 0.0556, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0833, + "mean_precision_at_10": 0.0056, + "zero_hit_rate": 0.8889, + "median_tokens_delivered": 16506.5, + "median_latency_ms": 23.393, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 9, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 17126, + "median_latency_ms": 23.707, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 19, + "mean_recall_at_5": 0.0263, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0526, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1184, + "full_recall_at_20": 0.0526, + "mean_recall_at_1000_tokens": 0.0263, + "mean_recall_at_4000_tokens": 0.0263, + "mean_recall_at_16000_tokens": 0.1053, + "mean_precision_at_10": 0.0105, + "zero_hit_rate": 0.7895, + "median_tokens_delivered": 17991, + "median_latency_ms": 21.563, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 80235.3, + "peak_rss_mb": 80242.7, + "delta_rss_mb": 7.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "agent-default", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 45959, + "latency_ms": 110.87, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [ + "lib/response.js", + "test/res.send.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 56953, + "latency_ms": 172.951, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 56945, + "latency_ms": 174.898, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 44.471, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 35724, + "latency_ms": 125.937, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 33168, + "latency_ms": 74.85, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27848, + "latency_ms": 80.483, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [ + "lib/response.js", + "test/res.redirect.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 33723, + "latency_ms": 121.891, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 42135, + "latency_ms": 110.593, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15002, + "latency_ms": 155.356, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 57447, + "latency_ms": 141.628, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 34150, + "latency_ms": 114.895, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 40896, + "latency_ms": 149.231, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34561, + "latency_ms": 185.629, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 47923, + "latency_ms": 195.806, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [ + "test/Router.js" + ], + "missed": [ + "lib/router/route.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 58965, + "latency_ms": 109.406, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [ + "lib/router/route.js", + "test/Route.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 35183, + "latency_ms": 207.013, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 45.688, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [ + "test/app.router.js" + ], + "missed": [ + "lib/router/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 54581, + "latency_ms": 178.538, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 36467, + "latency_ms": 119.855, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 44888, + "latency_ms": 108.639, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 50819, + "latency_ms": 138.168, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 33148, + "latency_ms": 102.271, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [ + "examples/search/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 53007, + "latency_ms": 176.077, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 45158, + "latency_ms": 131.911, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 26785, + "latency_ms": 109.851, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [], + "missed": [ + "test/app.router.js", + "test/res.format.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43299, + "latency_ms": 89.016, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 55255, + "latency_ms": 83.894, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 50259, + "latency_ms": 39.363, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [ + "lib/response.js", + "test/res.send.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 67935, + "latency_ms": 99.909, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 62554, + "latency_ms": 124.564, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 27.596, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 35724, + "latency_ms": 96.614, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 33425, + "latency_ms": 44.897, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27848, + "latency_ms": 39.059, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [ + "lib/response.js", + "test/res.redirect.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 32897, + "latency_ms": 92.347, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.cookie.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 45129, + "latency_ms": 76.778, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42759, + "latency_ms": 114.976, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 57739, + "latency_ms": 107.896, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 34150, + "latency_ms": 72.416, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 40896, + "latency_ms": 84.545, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34561, + "latency_ms": 111.961, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 55768, + "latency_ms": 101.895, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [ + "test/Router.js" + ], + "missed": [ + "lib/router/route.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 58965, + "latency_ms": 63.354, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [ + "lib/router/route.js", + "test/Route.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 45635, + "latency_ms": 132.325, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 29.081, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [ + "test/app.router.js" + ], + "missed": [ + "lib/router/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 54602, + "latency_ms": 87.524, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 36467, + "latency_ms": 88.708, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 44888, + "latency_ms": 76.975, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 51318, + "latency_ms": 84.618, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 33405, + "latency_ms": 53.775, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [ + "examples/search/index.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 61936, + "latency_ms": 103.166, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 45158, + "latency_ms": 104.056, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 27724, + "latency_ms": 68.763, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [], + "missed": [ + "test/app.router.js", + "test/res.format.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43299, + "latency_ms": 69.991, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 55255, + "latency_ms": 41.012, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2007, + "latency_ms": 50962.489, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [ + "lib/response.js", + "test/res.send.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1472, + "latency_ms": 10904.935, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [], + "missed": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5105.356, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 742, + "latency_ms": 18445.292, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1370, + "latency_ms": 4187.152, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2982, + "latency_ms": 3721.07, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1250, + "latency_ms": 3115.921, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [ + "lib/response.js", + "test/res.redirect.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1436, + "latency_ms": 4659.013, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1221, + "latency_ms": 3038.789, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 819, + "latency_ms": 1897.873, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 5215, + "latency_ms": 3597.191, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 9890, + "latency_ms": 2130.055, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1342, + "latency_ms": 2107.359, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1835.095, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [], + "missed": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6014, + "latency_ms": 2586.826, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [ + "lib/router/route.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1412, + "latency_ms": 3100.48, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [ + "lib/router/route.js", + "test/Route.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2275, + "latency_ms": 2157.029, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4919, + "latency_ms": 1545.718, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [ + "test/app.router.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1726, + "latency_ms": 1493.206, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 305, + "latency_ms": 1419.807, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1434, + "latency_ms": 1706.375, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1184, + "latency_ms": 1677.935, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3709, + "latency_ms": 1608.548, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [ + "examples/search/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2797, + "latency_ms": 1470.981, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 770, + "latency_ms": 1469.269, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 161, + "latency_ms": 1795.82, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [], + "missed": [ + "test/app.router.js", + "test/res.format.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 265, + "latency_ms": 1906.64, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 2698, + "latency_ms": 1503.39, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1148, + "latency_ms": 1809.901, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.send.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2445, + "latency_ms": 1312.94, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/application.js", + "lib/response.js" + ], + "missed": [ + "test/res.cookie.js" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 5687, + "latency_ms": 1436.942, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 8068, + "latency_ms": 1366.014, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1685, + "latency_ms": 1297.81, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1672, + "latency_ms": 1329.318, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3297, + "latency_ms": 1346.601, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [ + "test/res.redirect.js" + ], + "missed": [ + "lib/response.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2483, + "latency_ms": 1575.003, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1675, + "latency_ms": 1431.669, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 873, + "latency_ms": 1337.539, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 5239, + "latency_ms": 1445.419, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2710, + "latency_ms": 1395.184, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1369, + "latency_ms": 1394.449, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 805, + "latency_ms": 1334.631, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Router.js" + ], + "missed": [ + "test/Route.js" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 3, + "tokens_delivered": 5296, + "latency_ms": 1422.658, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [ + "lib/router/route.js" + ], + "missed": [ + "test/Router.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1464, + "latency_ms": 1342.111, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [ + "lib/router/route.js" + ], + "missed": [ + "test/Route.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2044, + "latency_ms": 1279.423, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1240, + "latency_ms": 1321.929, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [ + "lib/router/index.js", + "test/app.router.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 4647, + "latency_ms": 1295.993, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7, + "latency_ms": 1211.786, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1671, + "latency_ms": 1325.244, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1674, + "latency_ms": 1336.404, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1692, + "latency_ms": 1294.217, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [ + "examples/search/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1632, + "latency_ms": 1306.129, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3437, + "latency_ms": 1304.764, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4086, + "latency_ms": 1222.389, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [], + "missed": [ + "test/app.router.js", + "test/res.format.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1537, + "latency_ms": 1310.821, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1172, + "latency_ms": 1279.837, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 48438, + "latency_ms": 41.12, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [ + "lib/response.js", + "test/res.send.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 63857, + "latency_ms": 60.058, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 68922, + "latency_ms": 36.486, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [ + "lib/request.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 46929, + "latency_ms": 56.675, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 44158, + "latency_ms": 34.449, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 69122, + "latency_ms": 45.008, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48438, + "latency_ms": 42.187, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.redirect.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 56208, + "latency_ms": 43.651, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.cookie.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 54109, + "latency_ms": 39.993, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 69044, + "latency_ms": 40.313, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 57263, + "latency_ms": 47.779, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 48439, + "latency_ms": 40.853, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68691, + "latency_ms": 41.111, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 68940, + "latency_ms": 38.272, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js", + "test/Router.js" + ], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 57070, + "latency_ms": 48.733, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [ + "test/Router.js" + ], + "missed": [ + "lib/router/route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 55125, + "latency_ms": 34.578, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54998, + "latency_ms": 51.073, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 63660, + "latency_ms": 36.639, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [ + "lib/router/index.js", + "test/app.router.js" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 46930, + "latency_ms": 52.17, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63835, + "latency_ms": 37.533, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.jsonp.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 47132, + "latency_ms": 41.667, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 46929, + "latency_ms": 42.376, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 52216, + "latency_ms": 40.506, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [], + "missed": [ + "examples/search/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63752, + "latency_ms": 41.519, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [ + "lib/response.js" + ], + "missed": [ + "test/res.json.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 47164, + "latency_ms": 37.875, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 44146, + "latency_ms": 54.054, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [ + "test/app.router.js" + ], + "missed": [ + "test/res.format.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 69122, + "latency_ms": 37.571, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 52218, + "latency_ms": 54.208, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-05f40f4321fe", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15887, + "latency_ms": 32.101, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-18e5985b8a9d", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.send.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.send.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24167, + "latency_ms": 18.963, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-246f6f5aeeba", + "path_leak": false, + "required_files": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "found": [], + "missed": [ + "lib/application.js", + "lib/response.js", + "test/res.cookie.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17991, + "latency_ms": 18.7, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-353348a83e68", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15115, + "latency_ms": 20.966, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-3d10279826f5", + "path_leak": false, + "required_files": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.sendFile.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8352, + "latency_ms": 40.391, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-4c4f3ea10593", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25943, + "latency_ms": 18.721, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-4e61d0100d90", + "path_leak": false, + "required_files": [ + "test/res.type.js" + ], + "found": [], + "missed": [ + "test/res.type.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24105, + "latency_ms": 18.909, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-54271f69b511", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.redirect.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.redirect.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17989, + "latency_ms": 32.75, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-5855339455a7", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.cookie.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.cookie.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10199, + "latency_ms": 27.44, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-62336717bfb6", + "path_leak": true, + "required_files": [ + "examples/auth/views/head.ejs" + ], + "found": [], + "missed": [ + "examples/auth/views/head.ejs" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25824, + "latency_ms": 18.253, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-631ada0c645d", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/Router.js" + ], + "found": [ + "lib/router/index.js" + ], + "missed": [ + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 21567, + "latency_ms": 26.129, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-65b62065d2b4", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14979, + "latency_ms": 34.226, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-6cd404eb28ff", + "path_leak": true, + "required_files": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "found": [], + "missed": [ + "lib/request.js", + "test/req.acceptsCharsets.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21348, + "latency_ms": 20.444, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-6ed3439584b6", + "path_leak": false, + "required_files": [ + ".editorconfig" + ], + "found": [], + "missed": [ + ".editorconfig" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28836, + "latency_ms": 19.32, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-708ac4cdf5cd", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Route.js", + "test/Router.js" + ], + "found": [ + "test/Route.js" + ], + "missed": [ + "lib/router/index.js", + "lib/router/route.js", + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 25626, + "latency_ms": 35.738, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-74beeac0718c", + "path_leak": false, + "required_files": [ + "lib/router/route.js", + "test/Router.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26880, + "latency_ms": 23.854, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-7ec5dd2b3c5e", + "path_leak": true, + "required_files": [ + "lib/router/route.js", + "test/Route.js" + ], + "found": [], + "missed": [ + "lib/router/route.js", + "test/Route.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12752, + "latency_ms": 21.617, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-925a1dff1e42", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12150, + "latency_ms": 22.4, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-99a369f3d51b", + "path_leak": false, + "required_files": [ + "lib/router/index.js", + "test/app.router.js" + ], + "found": [], + "missed": [ + "lib/router/index.js", + "test/app.router.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12313, + "latency_ms": 27.433, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-9d8223d92ee8", + "path_leak": false, + "required_files": [ + "lib/request.js" + ], + "found": [], + "missed": [ + "lib/request.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14066, + "latency_ms": 20.324, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-9dd0e7afdb6d", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24966, + "latency_ms": 23.839, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-a1dbb113779c", + "path_leak": false, + "required_files": [ + "lib/response.js" + ], + "found": [], + "missed": [ + "lib/response.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18218, + "latency_ms": 19.848, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-b3906cbdded2", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 22036, + "latency_ms": 23.079, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-d12772393c82", + "path_leak": true, + "required_files": [ + "examples/search/index.js" + ], + "found": [], + "missed": [ + "examples/search/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17126, + "latency_ms": 24.665, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-f275e87dff1a", + "path_leak": true, + "required_files": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "found": [], + "missed": [ + "lib/response.js", + "test/res.json.js", + "test/res.jsonp.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15153, + "latency_ms": 22.636, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-f3fa758af966", + "path_leak": true, + "required_files": [ + "lib/router/index.js" + ], + "found": [], + "missed": [ + "lib/router/index.js" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15550, + "latency_ms": 23.707, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-f9954dd317a1", + "path_leak": false, + "required_files": [ + "test/app.router.js", + "test/res.format.js" + ], + "found": [ + "test/res.format.js" + ], + "missed": [ + "test/app.router.js" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 15695, + "latency_ms": 20.022, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "express-fed8c2a8857a", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10590, + "latency_ms": 21.563, + "indexed_revision_matches": true + } + ], + "tier": "small", + "tier_code_files": 141, + "tier_provenance": "Stamped after the fact from a measured file count; the run predates tier stamping. Tier is a property of the repository, not of the run." +} diff --git a/benchmarks/context-retrieval/results/score-pub-flask.json b/benchmarks/context-retrieval/results/score-pub-flask.json new file mode 100644 index 00000000..447b02f5 --- /dev/null +++ b/benchmarks/context-retrieval/results/score-pub-flask.json @@ -0,0 +1,6945 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "flask", + "head": "d318b683471101618febed18996405ad26462110" + }, + "corpus_counts": { + "cases": 39, + "path_leak": 7, + "baseline_blind": 32, + "multi_file": 17 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "providers": [ + { + "provider_id": "agent-default", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.2051, + "full_recall_at_5": 0.0769, + "mean_recall_at_10": 0.2927, + "full_recall_at_10": 0.1282, + "mean_recall_at_20": 0.3953, + "full_recall_at_20": 0.2308, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0128, + "mean_recall_at_16000_tokens": 0.1795, + "mean_precision_at_10": 0.0641, + "zero_hit_rate": 0.4615, + "median_tokens_delivered": 78305, + "median_latency_ms": 88.123, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 34, + "mean_recall_at_5": 0.1324, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1887, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3064, + "full_recall_at_20": 0.1176, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0147, + "mean_recall_at_16000_tokens": 0.1176, + "mean_precision_at_10": 0.0529, + "zero_hit_rate": 0.5294, + "median_tokens_delivered": 72042, + "median_latency_ms": 83.334, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 5, + "mean_recall_at_5": 0.7, + "full_recall_at_5": 0.6, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.6, + "mean_precision_at_10": 0.14, + "zero_hit_rate": 0, + "median_tokens_delivered": 89253, + "median_latency_ms": 101.714, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0.4762, + "full_recall_at_5": 0.2857, + "mean_recall_at_10": 0.4762, + "full_recall_at_10": 0.2857, + "mean_recall_at_20": 0.6667, + "full_recall_at_20": 0.4286, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.4286, + "mean_precision_at_10": 0.0857, + "zero_hit_rate": 0.1429, + "median_tokens_delivered": 88684, + "median_latency_ms": 95.365, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.1458, + "full_recall_at_5": 0.0313, + "mean_recall_at_10": 0.2526, + "full_recall_at_10": 0.0938, + "mean_recall_at_20": 0.3359, + "full_recall_at_20": 0.1875, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0156, + "mean_recall_at_16000_tokens": 0.125, + "mean_precision_at_10": 0.0594, + "zero_hit_rate": 0.5313, + "median_tokens_delivered": 76235.5, + "median_latency_ms": 80.5345, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 80230.6, + "peak_rss_mb": 80257, + "delta_rss_mb": 26.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "keyword-search", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.1667, + "full_recall_at_5": 0.0256, + "mean_recall_at_10": 0.3013, + "full_recall_at_10": 0.1282, + "mean_recall_at_20": 0.3953, + "full_recall_at_20": 0.2308, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0513, + "mean_recall_at_16000_tokens": 0.1282, + "mean_precision_at_10": 0.0667, + "zero_hit_rate": 0.4615, + "median_tokens_delivered": 80731, + "median_latency_ms": 46.196, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 34, + "mean_recall_at_5": 0.1324, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1985, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3064, + "full_recall_at_20": 0.1176, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0294, + "mean_recall_at_16000_tokens": 0.1029, + "mean_precision_at_10": 0.0559, + "zero_hit_rate": 0.5294, + "median_tokens_delivered": 78155, + "median_latency_ms": 41.997, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 5, + "mean_recall_at_5": 0.4, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2, + "mean_recall_at_16000_tokens": 0.3, + "mean_precision_at_10": 0.14, + "zero_hit_rate": 0, + "median_tokens_delivered": 89253, + "median_latency_ms": 61.452, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0.1905, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.5238, + "full_recall_at_10": 0.2857, + "mean_recall_at_20": 0.5238, + "full_recall_at_20": 0.2857, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.1429, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 87992, + "median_latency_ms": 56.206, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.1615, + "full_recall_at_5": 0.0313, + "mean_recall_at_10": 0.2526, + "full_recall_at_10": 0.0938, + "mean_recall_at_20": 0.3672, + "full_recall_at_20": 0.2188, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0625, + "mean_recall_at_16000_tokens": 0.125, + "mean_precision_at_10": 0.0594, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 78701.5, + "median_latency_ms": 41.997, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 79420.6, + "peak_rss_mb": 79493.8, + "delta_rss_mb": 73.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codesearch", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.2906, + "full_recall_at_5": 0.1795, + "mean_recall_at_10": 0.297, + "full_recall_at_10": 0.1795, + "mean_recall_at_20": 0.3226, + "full_recall_at_20": 0.2051, + "mean_recall_at_1000_tokens": 0.2436, + "mean_recall_at_4000_tokens": 0.3226, + "mean_recall_at_16000_tokens": 0.3226, + "mean_precision_at_10": 0.0746, + "zero_hit_rate": 0.5641, + "median_tokens_delivered": 1538, + "median_latency_ms": 1374.292, + "payload_kind": [ + "chunks" + ], + "unavailable": 2 + }, + "baseline_missed": { + "cases": 34, + "mean_recall_at_5": 0.2892, + "full_recall_at_5": 0.1765, + "mean_recall_at_10": 0.2966, + "full_recall_at_10": 0.1765, + "mean_recall_at_20": 0.2966, + "full_recall_at_20": 0.1765, + "mean_recall_at_1000_tokens": 0.2353, + "mean_recall_at_4000_tokens": 0.2966, + "mean_recall_at_16000_tokens": 0.2966, + "mean_precision_at_10": 0.072, + "zero_hit_rate": 0.5882, + "median_tokens_delivered": 1578, + "median_latency_ms": 1369.3995, + "payload_kind": [ + "chunks" + ], + "unavailable": 1 + }, + "baseline_solved": { + "cases": 5, + "mean_recall_at_5": 0.3, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.3, + "full_recall_at_10": 0.2, + "mean_recall_at_20": 0.5, + "full_recall_at_20": 0.4, + "mean_recall_at_1000_tokens": 0.3, + "mean_recall_at_4000_tokens": 0.5, + "mean_recall_at_16000_tokens": 0.5, + "mean_precision_at_10": 0.0917, + "zero_hit_rate": 0.4, + "median_tokens_delivered": 1043, + "median_latency_ms": 1658.895, + "payload_kind": [ + "chunks" + ], + "unavailable": 1 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0.3333, + "full_recall_at_5": 0.2857, + "mean_recall_at_10": 0.3333, + "full_recall_at_10": 0.2857, + "mean_recall_at_20": 0.3333, + "full_recall_at_20": 0.2857, + "mean_recall_at_1000_tokens": 0.3333, + "mean_recall_at_4000_tokens": 0.3333, + "mean_recall_at_16000_tokens": 0.3333, + "mean_precision_at_10": 0.0778, + "zero_hit_rate": 0.5714, + "median_tokens_delivered": 1400, + "median_latency_ms": 1658.895, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.2813, + "full_recall_at_5": 0.1563, + "mean_recall_at_10": 0.2891, + "full_recall_at_10": 0.1563, + "mean_recall_at_20": 0.3203, + "full_recall_at_20": 0.1875, + "mean_recall_at_1000_tokens": 0.224, + "mean_recall_at_4000_tokens": 0.3203, + "mean_recall_at_16000_tokens": 0.3203, + "mean_precision_at_10": 0.0739, + "zero_hit_rate": 0.5625, + "median_tokens_delivered": 1618, + "median_latency_ms": 1351.7755, + "payload_kind": [ + "chunks" + ], + "unavailable": 2 + } + }, + "resources": { + "baseline_rss_mb": 79494.8, + "peak_rss_mb": 82334.1, + "delta_rss_mb": 2839.3, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.2714, + "full_recall_at_5": 0.1026, + "mean_recall_at_10": 0.312, + "full_recall_at_10": 0.1282, + "mean_recall_at_20": 0.3547, + "full_recall_at_20": 0.1795, + "mean_recall_at_1000_tokens": 0.2863, + "mean_recall_at_4000_tokens": 0.3547, + "mean_recall_at_16000_tokens": 0.3547, + "mean_precision_at_10": 0.0765, + "zero_hit_rate": 0.4872, + "median_tokens_delivered": 1702, + "median_latency_ms": 1445.37, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 34, + "mean_recall_at_5": 0.2672, + "full_recall_at_5": 0.0882, + "mean_recall_at_10": 0.3137, + "full_recall_at_10": 0.1176, + "mean_recall_at_20": 0.3333, + "full_recall_at_20": 0.1471, + "mean_recall_at_1000_tokens": 0.2843, + "mean_recall_at_4000_tokens": 0.3333, + "mean_recall_at_16000_tokens": 0.3333, + "mean_precision_at_10": 0.0819, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 1712, + "median_latency_ms": 1463.302, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 5, + "mean_recall_at_5": 0.3, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.3, + "full_recall_at_10": 0.2, + "mean_recall_at_20": 0.5, + "full_recall_at_20": 0.4, + "mean_recall_at_1000_tokens": 0.3, + "mean_recall_at_4000_tokens": 0.5, + "mean_recall_at_16000_tokens": 0.5, + "mean_precision_at_10": 0.04, + "zero_hit_rate": 0.4, + "median_tokens_delivered": 1687, + "median_latency_ms": 1403.222, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0.4524, + "full_recall_at_5": 0.1429, + "mean_recall_at_10": 0.4524, + "full_recall_at_10": 0.1429, + "mean_recall_at_20": 0.4524, + "full_recall_at_20": 0.1429, + "mean_recall_at_1000_tokens": 0.4524, + "mean_recall_at_4000_tokens": 0.4524, + "mean_recall_at_16000_tokens": 0.4524, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 1689, + "median_latency_ms": 1423.875, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.2318, + "full_recall_at_5": 0.0938, + "mean_recall_at_10": 0.2813, + "full_recall_at_10": 0.125, + "mean_recall_at_20": 0.3333, + "full_recall_at_20": 0.1875, + "mean_recall_at_1000_tokens": 0.25, + "mean_recall_at_4000_tokens": 0.3333, + "mean_recall_at_16000_tokens": 0.3333, + "mean_precision_at_10": 0.0714, + "zero_hit_rate": 0.5313, + "median_tokens_delivered": 2131, + "median_latency_ms": 1446.001, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 78418.5, + "peak_rss_mb": 79195, + "delta_rss_mb": 776.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 48340992 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.2671, + "full_recall_at_5": 0.1795, + "mean_recall_at_10": 0.4573, + "full_recall_at_10": 0.3077, + "mean_recall_at_20": 0.6026, + "full_recall_at_20": 0.4359, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0321, + "mean_recall_at_16000_tokens": 0.1603, + "mean_precision_at_10": 0.0718, + "zero_hit_rate": 0.2308, + "median_tokens_delivered": 80328, + "median_latency_ms": 24.656, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 34, + "mean_recall_at_5": 0.2475, + "full_recall_at_5": 0.1765, + "mean_recall_at_10": 0.4216, + "full_recall_at_10": 0.2647, + "mean_recall_at_20": 0.5735, + "full_recall_at_20": 0.3824, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0368, + "mean_recall_at_16000_tokens": 0.1397, + "mean_precision_at_10": 0.0676, + "zero_hit_rate": 0.2353, + "median_tokens_delivered": 80321, + "median_latency_ms": 24.8585, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 5, + "mean_recall_at_5": 0.4, + "full_recall_at_5": 0.2, + "mean_recall_at_10": 0.7, + "full_recall_at_10": 0.6, + "mean_recall_at_20": 0.8, + "full_recall_at_20": 0.8, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.3, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.2, + "median_tokens_delivered": 81351, + "median_latency_ms": 23.923, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0.1429, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.4048, + "full_recall_at_10": 0.1429, + "mean_recall_at_20": 0.4762, + "full_recall_at_20": 0.2857, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.0714, + "zero_hit_rate": 0.2857, + "median_tokens_delivered": 98043, + "median_latency_ms": 24.288, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.2943, + "full_recall_at_5": 0.2188, + "mean_recall_at_10": 0.4688, + "full_recall_at_10": 0.3438, + "mean_recall_at_20": 0.6302, + "full_recall_at_20": 0.4688, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0391, + "mean_recall_at_16000_tokens": 0.1953, + "mean_precision_at_10": 0.0719, + "zero_hit_rate": 0.2188, + "median_tokens_delivered": 80264.5, + "median_latency_ms": 24.713, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 75347, + "peak_rss_mb": 75347, + "delta_rss_mb": 0, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 39, + "mean_recall_at_5": 0.0427, + "full_recall_at_5": 0.0256, + "mean_recall_at_10": 0.0962, + "full_recall_at_10": 0.0513, + "mean_recall_at_20": 0.1859, + "full_recall_at_20": 0.1026, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0171, + "mean_recall_at_16000_tokens": 0.094, + "mean_precision_at_10": 0.0179, + "zero_hit_rate": 0.6667, + "median_tokens_delivered": 30503, + "median_latency_ms": 10.322, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 34, + "mean_recall_at_5": 0.049, + "full_recall_at_5": 0.0294, + "mean_recall_at_10": 0.0956, + "full_recall_at_10": 0.0588, + "mean_recall_at_20": 0.1544, + "full_recall_at_20": 0.0882, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0196, + "mean_recall_at_16000_tokens": 0.0784, + "mean_precision_at_10": 0.0176, + "zero_hit_rate": 0.7059, + "median_tokens_delivered": 30323.5, + "median_latency_ms": 10.338, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 5, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.4, + "full_recall_at_20": 0.2, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.2, + "mean_precision_at_10": 0.02, + "zero_hit_rate": 0.4, + "median_tokens_delivered": 32720, + "median_latency_ms": 10.191, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 7, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0476, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.119, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.0714, + "mean_precision_at_10": 0.0143, + "zero_hit_rate": 0.7143, + "median_tokens_delivered": 26267, + "median_latency_ms": 10.322, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 32, + "mean_recall_at_5": 0.0521, + "full_recall_at_5": 0.0313, + "mean_recall_at_10": 0.1068, + "full_recall_at_10": 0.0625, + "mean_recall_at_20": 0.2005, + "full_recall_at_20": 0.125, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0208, + "mean_recall_at_16000_tokens": 0.099, + "mean_precision_at_10": 0.0188, + "zero_hit_rate": 0.6563, + "median_tokens_delivered": 31713, + "median_latency_ms": 10.3265, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 74733, + "peak_rss_mb": 74756, + "delta_rss_mb": 23, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "agent-default", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54022, + "latency_ms": 107.732, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 91167, + "latency_ms": 79.377, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 89842, + "latency_ms": 76.082, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47778, + "latency_ms": 112.571, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [ + "src/flask/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 88842, + "latency_ms": 94.415, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4578, + "latency_ms": 73.775, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_cli.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 94107, + "latency_ms": 103.765, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 85655, + "latency_ms": 128.309, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + ".flake8", + "tests/test_appctx.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 76672, + "latency_ms": 89.288, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py" + ], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88684, + "latency_ms": 95.365, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86634, + "latency_ms": 67.92, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 64792, + "latency_ms": 81.692, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 95646, + "latency_ms": 107.272, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [ + "src/flask/testing.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 82763, + "latency_ms": 112.155, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 75799, + "latency_ms": 101.015, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 89253, + "latency_ms": 96.937, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [], + "missed": [ + "src/flask/config.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 44.894, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 78305, + "latency_ms": 177.313, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [ + "src/flask/typing.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 85957, + "latency_ms": 114.094, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 58.569, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4578, + "latency_ms": 86.706, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4462, + "latency_ms": 84.976, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [], + "missed": [ + "pyproject.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80731, + "latency_ms": 68.778, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4578, + "latency_ms": 58.082, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [], + "missed": [ + ".pre-commit-config.yaml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 43.188, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [ + "src/flask/wrappers.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 98117, + "latency_ms": 111.994, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 107434, + "latency_ms": 75.959, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 99836, + "latency_ms": 95.019, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 59.402, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4578, + "latency_ms": 77.965, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 94980, + "latency_ms": 101.714, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 67412, + "latency_ms": 88.123, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [ + "examples/javascript/tests/test_js_example.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 46017, + "latency_ms": 90.46, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4578, + "latency_ms": 71.62, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 49.655, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 1, + "tokens_delivered": 94787, + "latency_ms": 90.527, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4578, + "latency_ms": 66.342, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [ + "src/flask/sansio/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 93633, + "latency_ms": 73.798, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [ + "src/flask/json/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 89863, + "latency_ms": 92.604, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54022, + "latency_ms": 57.498, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 91167, + "latency_ms": 40.677, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 89842, + "latency_ms": 39.576, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 88384, + "latency_ms": 68.608, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [ + "src/flask/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 54911, + "latency_ms": 46.196, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4578, + "latency_ms": 34.866, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_cli.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 94107, + "latency_ms": 64.489, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 85655, + "latency_ms": 91.244, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + ".flake8", + "tests/test_appctx.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 76672, + "latency_ms": 45.835, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py" + ], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 92485, + "latency_ms": 47.871, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86634, + "latency_ms": 35.453, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 64792, + "latency_ms": 32.984, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 95646, + "latency_ms": 59.63, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [ + "src/flask/testing.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 84363, + "latency_ms": 65.84, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 76631, + "latency_ms": 59.98, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 89253, + "latency_ms": 61.452, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [], + "missed": [ + "src/flask/config.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 26.542, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 100786, + "latency_ms": 123.679, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [ + "src/flask/typing.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 87992, + "latency_ms": 61.021, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 39.56, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4578, + "latency_ms": 36.296, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4462, + "latency_ms": 32.724, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [], + "missed": [ + "pyproject.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80731, + "latency_ms": 47.385, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4578, + "latency_ms": 34.238, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [], + "missed": [ + ".pre-commit-config.yaml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 24.575, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [ + "src/flask/wrappers.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 98117, + "latency_ms": 93.808, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 107434, + "latency_ms": 36.223, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 99836, + "latency_ms": 69.8, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 43.317, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4578, + "latency_ms": 35.716, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 94980, + "latency_ms": 82.202, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 67412, + "latency_ms": 46.613, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [], + "missed": [ + "examples/javascript/tests/test_js_example.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 79638, + "latency_ms": 39.268, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4578, + "latency_ms": 34.377, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 26.009, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 1, + "tokens_delivered": 94787, + "latency_ms": 65.137, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4578, + "latency_ms": 34.769, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [ + "src/flask/sansio/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 93633, + "latency_ms": 52.774, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [ + "src/flask/json/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 89863, + "latency_ms": 56.206, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1832, + "latency_ms": 38088.439, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3569, + "latency_ms": 3770.123, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1041, + "latency_ms": 10139.306, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 502, + "latency_ms": 3597.77, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 407, + "latency_ms": 1364.507, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 527, + "latency_ms": 9579.318, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1389, + "latency_ms": 2731.238, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 457, + "latency_ms": 3076.69, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + ".flake8", + "tests/test_appctx.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 1831, + "latency_ms": 2674.764, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "tests/test_instance_config.py" + ], + "missed": [ + "src/flask/scaffold.py", + "tox.ini" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1111, + "recall_at_20": 0.3333, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 1491, + "latency_ms": 7158.296, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 826, + "latency_ms": 1374.292, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "tests/test_reqctx.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1151, + "latency_ms": 6544.999, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.4286, + "recall_at_20": 0.75, + "precision_at_20": 0.4286, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 1, + "tokens_delivered": 1384, + "latency_ms": 3734.492, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2349.148, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1842.353, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1043, + "latency_ms": 7907.477, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [], + "missed": [ + "src/flask/config.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1538, + "latency_ms": 1694.036, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1634, + "latency_ms": 1258.707, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [], + "missed": [ + "src/flask/typing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1025, + "latency_ms": 1658.895, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1618, + "latency_ms": 1138.543, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2081, + "latency_ms": 1117.145, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2344, + "latency_ms": 1033.494, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [ + "pyproject.toml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2355, + "latency_ms": 2042.235, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2071, + "latency_ms": 1089.007, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [], + "missed": [ + ".pre-commit-config.yaml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1430, + "latency_ms": 1329.259, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [ + "src/flask/wrappers.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 13, + "tokens_delivered": 2472, + "latency_ms": 1097.086, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4703, + "latency_ms": 1066.682, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 2796, + "latency_ms": 1091.204, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1618, + "latency_ms": 931.873, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 527, + "latency_ms": 997.067, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2075, + "latency_ms": 946.165, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4904, + "latency_ms": 1071.166, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [ + "examples/javascript/tests/test_js_example.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1272, + "latency_ms": 1114.785, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 527, + "latency_ms": 1576.227, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2643, + "latency_ms": 965.147, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_views.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py", + "tests/test_cli.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.5, + "precision_at_20": 0.3, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3685, + "latency_ms": 1033.749, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1694, + "latency_ms": 1032.932, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [ + "src/flask/sansio/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3486, + "latency_ms": 990.909, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1400, + "latency_ms": 2293.361, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1694, + "latency_ms": 1975.285, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1678, + "latency_ms": 1860.063, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7470, + "latency_ms": 1914.841, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1672, + "latency_ms": 1848.026, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [ + "src/flask/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1650, + "latency_ms": 1944.202, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [ + "src/flask/json/provider.py" + ], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2550, + "latency_ms": 1926.087, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_cli.py" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 285, + "latency_ms": 1932.368, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 4860, + "latency_ms": 1682.259, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [], + "missed": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3947, + "latency_ms": 1567.659, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py" + ], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 1710, + "latency_ms": 1762.418, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1677, + "latency_ms": 1624.887, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1676, + "latency_ms": 1513.152, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1690, + "latency_ms": 1622.101, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [ + "src/flask/testing.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1712, + "latency_ms": 1565.674, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1680, + "latency_ms": 1401.028, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1676, + "latency_ms": 1403.222, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [], + "missed": [ + "src/flask/config.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2610, + "latency_ms": 1416.681, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 799, + "latency_ms": 1461.691, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [], + "missed": [ + "src/flask/typing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1689, + "latency_ms": 1423.875, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1428.002, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1469.913, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1445.37, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [], + "missed": [ + "pyproject.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2610, + "latency_ms": 1432.889, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1441.881, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [], + "missed": [ + ".pre-commit-config.yaml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1464.913, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [ + "src/flask/wrappers.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 1687, + "latency_ms": 1419.283, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1692, + "latency_ms": 1446.632, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 1712, + "latency_ms": 1516.167, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1441.519, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1443.424, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/blueprints.py" + ], + "missed": [ + "tests/test_blueprints.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1696, + "latency_ms": 1099.158, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1675, + "latency_ms": 1097.63, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [ + "examples/javascript/tests/test_js_example.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1682, + "latency_ms": 1094.791, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1104.022, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1088.308, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_views.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_cli.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.6667, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 1702, + "latency_ms": 1099.398, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2550, + "latency_ms": 1104.117, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [ + "src/flask/sansio/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1669, + "latency_ms": 1115.349, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [ + "src/flask/json/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1666, + "latency_ms": 1212.273, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 98958, + "latency_ms": 31.674, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [ + "src/flask/ctx.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 98182, + "latency_ms": 25.842, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 97356, + "latency_ms": 26.05, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 100544, + "latency_ms": 24.947, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [ + "src/flask/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 97421, + "latency_ms": 25.104, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [ + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 70486, + "latency_ms": 26.399, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "src/flask/cli.py", + "tests/test_cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 70069, + "latency_ms": 24.989, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [ + "src/flask/cli.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 76284, + "latency_ms": 25.514, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + ".flake8", + "tests/test_appctx.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 101135, + "latency_ms": 24.571, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py" + ], + "missed": [ + "tests/test_instance_config.py", + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 98043, + "latency_ms": 28.274, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70565, + "latency_ms": 26.389, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "tests/test_reqctx.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 100513, + "latency_ms": 27.304, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 72596, + "latency_ms": 25.179, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [ + "src/flask/testing.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 73072, + "latency_ms": 24.513, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 96806, + "latency_ms": 23.261, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 81351, + "latency_ms": 25.984, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [], + "missed": [ + "src/flask/config.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 99251, + "latency_ms": 23.401, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70565, + "latency_ms": 24.467, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [ + "src/flask/typing.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 75828, + "latency_ms": 23.671, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 96787, + "latency_ms": 24.77, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 70795, + "latency_ms": 24.202, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 97421, + "latency_ms": 24.337, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [ + "pyproject.toml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 80328, + "latency_ms": 24.145, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 70553, + "latency_ms": 25.041, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [ + ".pre-commit-config.yaml" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 80215, + "latency_ms": 25.207, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [], + "missed": [ + "src/flask/wrappers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70730, + "latency_ms": 24.554, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 99308, + "latency_ms": 23.851, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 97261, + "latency_ms": 23.631, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 95560, + "latency_ms": 23.737, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [ + "src/flask/sessions.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 70731, + "latency_ms": 25.233, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 100533, + "latency_ms": 23.923, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 81260, + "latency_ms": 25.648, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [], + "missed": [ + "examples/javascript/tests/test_js_example.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70553, + "latency_ms": 24.288, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 71520, + "latency_ms": 24.656, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70565, + "latency_ms": 25.34, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_cli.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_blueprints.py", + "tests/test_views.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 80314, + "latency_ms": 24.521, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [ + "src/flask/sessions.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 70461, + "latency_ms": 24.654, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70069, + "latency_ms": 23.327, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [ + "src/flask/json/__init__.py" + ], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 98082, + "latency_ms": 23.058, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-03b410066b96", + "path_leak": true, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17705, + "latency_ms": 11.643, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-095651be9eec", + "path_leak": false, + "required_files": [ + "src/flask/ctx.py" + ], + "found": [], + "missed": [ + "src/flask/ctx.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22208, + "latency_ms": 11.305, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-0afeb1d11cad", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30144, + "latency_ms": 11.455, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-0d8c8ba71bc6", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30503, + "latency_ms": 10.157, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-0ec9192cf25f", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/__init__.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 11009, + "latency_ms": 10.322, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-176fdfa00023", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py", + "src/flask/json/provider.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27983, + "latency_ms": 10.416, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-1af8f95785b8", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py", + "tests/test_cli.py" + ], + "found": [ + "tests/test_cli.py" + ], + "missed": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 26044, + "latency_ms": 10.354, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-1d5abfadd713", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/cli.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40575, + "latency_ms": 10.37, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-261e4a6cf287", + "path_leak": false, + "required_files": [ + ".flake8", + "tests/test_appctx.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_appctx.py" + ], + "missed": [ + ".flake8", + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 37624, + "latency_ms": 9.762, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-3ba37d2afe65", + "path_leak": true, + "required_files": [ + "src/flask/scaffold.py", + "tests/test_instance_config.py", + "tox.ini" + ], + "found": [ + "src/flask/scaffold.py" + ], + "missed": [ + "tests/test_instance_config.py", + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 62976, + "latency_ms": 9.51, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-40b78fa2ea90", + "path_leak": false, + "required_files": [ + "tox.ini" + ], + "found": [], + "missed": [ + "tox.ini" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36940, + "latency_ms": 10.842, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-426a1e25b77e", + "path_leak": false, + "required_files": [ + "tests/test_basic.py", + "tests/test_reqctx.py" + ], + "found": [ + "tests/test_reqctx.py" + ], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 18661, + "latency_ms": 10.299, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-4995a775df21", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [ + "CHANGES.rst", + "tests/test_basic.py", + "tests/test_blueprints.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 50380, + "latency_ms": 10.235, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-54c3f87af9f6", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/testing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30706, + "latency_ms": 10.678, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-6f79cb8a2346", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [ + "src/flask/app.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 45986, + "latency_ms": 9.424, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-7203feabf723", + "path_leak": false, + "required_files": [ + "src/flask/app.py", + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 32720, + "latency_ms": 10.191, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-746455d1038f", + "path_leak": false, + "required_files": [ + "src/flask/config.py" + ], + "found": [ + "src/flask/config.py" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 28009, + "latency_ms": 9.599, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-74721b48f034", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36384, + "latency_ms": 9.698, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-80cf589a26d7", + "path_leak": true, + "required_files": [ + "src/flask/typing.py" + ], + "found": [], + "missed": [ + "src/flask/typing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30334, + "latency_ms": 10.584, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-826514b8eb18", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [], + "missed": [ + "tests/test_basic.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28572, + "latency_ms": 9.412, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-860a25c390eb", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43383, + "latency_ms": 10.908, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-9532cba45d23", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19148, + "latency_ms": 10.827, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-976459f7cb35", + "path_leak": false, + "required_files": [ + "pyproject.toml" + ], + "found": [], + "missed": [ + "pyproject.toml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33010, + "latency_ms": 10.132, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-98ae71897600", + "path_leak": false, + "required_files": [ + "src/flask/app.py" + ], + "found": [], + "missed": [ + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25252, + "latency_ms": 10.864, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-9b74a90dd3c4", + "path_leak": false, + "required_files": [ + ".pre-commit-config.yaml" + ], + "found": [ + ".pre-commit-config.yaml" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 25471, + "latency_ms": 9.767, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-a363642a3288", + "path_leak": false, + "required_files": [ + "src/flask/wrappers.py" + ], + "found": [], + "missed": [ + "src/flask/wrappers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45386, + "latency_ms": 11.574, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-a6a7a57380cd", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/app.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23746, + "latency_ms": 11.153, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-bda295d37fa3", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/helpers.py", + "src/flask/scaffold.py" + ], + "found": [ + "src/flask/helpers.py" + ], + "missed": [ + "CHANGES.rst", + "src/flask/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 36859, + "latency_ms": 10.408, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-c49ce2e1eb8b", + "path_leak": false, + "required_files": [ + "tests/test_basic.py" + ], + "found": [ + "tests/test_basic.py" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 36829, + "latency_ms": 11.095, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-ccf125bf3010", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 51711, + "latency_ms": 10.453, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-d7b6c1f6703d", + "path_leak": true, + "required_files": [ + "src/flask/blueprints.py", + "tests/test_blueprints.py" + ], + "found": [ + "tests/test_blueprints.py" + ], + "missed": [ + "src/flask/blueprints.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 26267, + "latency_ms": 9.543, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-de8429ffda8c", + "path_leak": false, + "required_files": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "found": [], + "missed": [ + "src/flask/testing.py", + "tests/test_testing.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23985, + "latency_ms": 9.873, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-df201ed152b6", + "path_leak": true, + "required_files": [ + "examples/javascript/tests/test_js_example.py" + ], + "found": [], + "missed": [ + "examples/javascript/tests/test_js_example.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40972, + "latency_ms": 9.62, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-dffe3034825e", + "path_leak": false, + "required_files": [ + "src/flask/helpers.py" + ], + "found": [], + "missed": [ + "src/flask/helpers.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41679, + "latency_ms": 9.96, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-e63ead4208c1", + "path_leak": false, + "required_files": [ + "CHANGES.rst" + ], + "found": [], + "missed": [ + "CHANGES.rst" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35855, + "latency_ms": 9.873, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-e82db2ca3a22", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py", + "tests/test_views.py" + ], + "found": [ + "tests/test_views.py" + ], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/app.py", + "tests/test_basic.py", + "tests/test_blueprints.py", + "tests/test_cli.py" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.1667, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 2, + "tokens_delivered": 27943, + "latency_ms": 9.639, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-eb1182a10e75", + "path_leak": false, + "required_files": [ + "src/flask/sessions.py" + ], + "found": [], + "missed": [ + "src/flask/sessions.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19505, + "latency_ms": 10.841, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-ecc057dd4857", + "path_leak": false, + "required_files": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/sansio/scaffold.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 42482, + "latency_ms": 10.08, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "flask-eede1a3685e2", + "path_leak": true, + "required_files": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "found": [], + "missed": [ + "CHANGES.rst", + "src/flask/json/__init__.py" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25042, + "latency_ms": 11.466, + "indexed_revision_matches": true + } + ], + "tier": "small", + "tier_code_files": 83, + "tier_provenance": "Stamped after the fact from a measured file count; the run predates tier stamping. Tier is a property of the repository, not of the run." +} diff --git a/benchmarks/context-retrieval/results/score-pub-gin.json b/benchmarks/context-retrieval/results/score-pub-gin.json new file mode 100644 index 00000000..85a83421 --- /dev/null +++ b/benchmarks/context-retrieval/results/score-pub-gin.json @@ -0,0 +1,12985 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "corpus_counts": { + "cases": 76, + "path_leak": 23, + "baseline_blind": 53, + "multi_file": 54 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "providers": [ + { + "provider_id": "agent-default", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.6042, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.7566, + "full_recall_at_10": 0.6842, + "mean_recall_at_20": 0.7917, + "full_recall_at_20": 0.7368, + "mean_recall_at_1000_tokens": 0.0066, + "mean_recall_at_4000_tokens": 0.1557, + "mean_recall_at_16000_tokens": 0.5143, + "mean_precision_at_10": 0.1632, + "zero_hit_rate": 0.1579, + "median_tokens_delivered": 67885.5, + "median_latency_ms": 70.477, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.1563, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2292, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3403, + "full_recall_at_20": 0.1667, + "mean_recall_at_1000_tokens": 0.0208, + "mean_recall_at_4000_tokens": 0.0833, + "mean_recall_at_16000_tokens": 0.1563, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 67258.5, + "median_latency_ms": 57.411, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.8109, + "full_recall_at_5": 0.7308, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1891, + "mean_recall_at_16000_tokens": 0.6795, + "mean_precision_at_10": 0.1923, + "zero_hit_rate": 0, + "median_tokens_delivered": 69842, + "median_latency_ms": 71.4935, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.7826, + "full_recall_at_5": 0.6522, + "mean_recall_at_10": 0.9058, + "full_recall_at_10": 0.8261, + "mean_recall_at_20": 0.9275, + "full_recall_at_20": 0.8696, + "mean_recall_at_1000_tokens": 0.0217, + "mean_recall_at_4000_tokens": 0.3768, + "mean_recall_at_16000_tokens": 0.8261, + "mean_precision_at_10": 0.1913, + "zero_hit_rate": 0, + "median_tokens_delivered": 63498, + "median_latency_ms": 71.356, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.5267, + "full_recall_at_5": 0.434, + "mean_recall_at_10": 0.6918, + "full_recall_at_10": 0.6226, + "mean_recall_at_20": 0.7327, + "full_recall_at_20": 0.6792, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0597, + "mean_recall_at_16000_tokens": 0.3789, + "mean_precision_at_10": 0.1509, + "zero_hit_rate": 0.2264, + "median_tokens_delivered": 71198, + "median_latency_ms": 68.793, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 78205.1, + "peak_rss_mb": 78282.7, + "delta_rss_mb": 77.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "keyword-search", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.591, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.7325, + "full_recall_at_10": 0.6447, + "mean_recall_at_20": 0.7917, + "full_recall_at_20": 0.7237, + "mean_recall_at_1000_tokens": 0.0066, + "mean_recall_at_4000_tokens": 0.1118, + "mean_recall_at_16000_tokens": 0.5296, + "mean_precision_at_10": 0.1579, + "zero_hit_rate": 0.1579, + "median_tokens_delivered": 70681, + "median_latency_ms": 44.153, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.1493, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2431, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3542, + "full_recall_at_20": 0.1667, + "mean_recall_at_1000_tokens": 0.0208, + "mean_recall_at_4000_tokens": 0.0417, + "mean_recall_at_16000_tokens": 0.1493, + "mean_precision_at_10": 0.1042, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 67270.5, + "median_latency_ms": 36.3085, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.7949, + "full_recall_at_5": 0.7308, + "mean_recall_at_10": 0.9583, + "full_recall_at_10": 0.9423, + "mean_recall_at_20": 0.9936, + "full_recall_at_20": 0.9808, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1442, + "mean_recall_at_16000_tokens": 0.7051, + "mean_precision_at_10": 0.1827, + "zero_hit_rate": 0, + "median_tokens_delivered": 71145, + "median_latency_ms": 45.5485, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.6522, + "full_recall_at_5": 0.4783, + "mean_recall_at_10": 0.8261, + "full_recall_at_10": 0.6957, + "mean_recall_at_20": 0.9275, + "full_recall_at_20": 0.8261, + "mean_recall_at_1000_tokens": 0.0217, + "mean_recall_at_4000_tokens": 0.1522, + "mean_recall_at_16000_tokens": 0.6377, + "mean_precision_at_10": 0.1739, + "zero_hit_rate": 0, + "median_tokens_delivered": 69644, + "median_latency_ms": 44.778, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.5645, + "full_recall_at_5": 0.5094, + "mean_recall_at_10": 0.6918, + "full_recall_at_10": 0.6226, + "mean_recall_at_20": 0.7327, + "full_recall_at_20": 0.6792, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0943, + "mean_recall_at_16000_tokens": 0.4827, + "mean_precision_at_10": 0.1509, + "zero_hit_rate": 0.2264, + "median_tokens_delivered": 71198, + "median_latency_ms": 43.939, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 78273.3, + "peak_rss_mb": 78384, + "delta_rss_mb": 110.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codesearch", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.5099, + "full_recall_at_5": 0.3816, + "mean_recall_at_10": 0.5921, + "full_recall_at_10": 0.4737, + "mean_recall_at_20": 0.5921, + "full_recall_at_20": 0.4737, + "mean_recall_at_1000_tokens": 0.5099, + "mean_recall_at_4000_tokens": 0.5921, + "mean_recall_at_16000_tokens": 0.5921, + "mean_precision_at_10": 0.1692, + "zero_hit_rate": 0.2895, + "median_tokens_delivered": 1184, + "median_latency_ms": 942.651, + "payload_kind": [ + "chunks" + ], + "unavailable": 5 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.1632, + "full_recall_at_5": 0.0417, + "mean_recall_at_10": 0.2292, + "full_recall_at_10": 0.0833, + "mean_recall_at_20": 0.2292, + "full_recall_at_20": 0.0833, + "mean_recall_at_1000_tokens": 0.184, + "mean_recall_at_4000_tokens": 0.2292, + "mean_recall_at_16000_tokens": 0.2292, + "mean_precision_at_10": 0.0989, + "zero_hit_rate": 0.625, + "median_tokens_delivered": 1224, + "median_latency_ms": 880.4995, + "payload_kind": [ + "chunks" + ], + "unavailable": 2 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.6699, + "full_recall_at_5": 0.5385, + "mean_recall_at_10": 0.7596, + "full_recall_at_10": 0.6538, + "mean_recall_at_20": 0.7596, + "full_recall_at_20": 0.6538, + "mean_recall_at_1000_tokens": 0.6603, + "mean_recall_at_4000_tokens": 0.7596, + "mean_recall_at_16000_tokens": 0.7596, + "mean_precision_at_10": 0.2016, + "zero_hit_rate": 0.1346, + "median_tokens_delivered": 1093.5, + "median_latency_ms": 998.094, + "payload_kind": [ + "chunks" + ], + "unavailable": 3 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.6377, + "full_recall_at_5": 0.4783, + "mean_recall_at_10": 0.8116, + "full_recall_at_10": 0.6957, + "mean_recall_at_20": 0.8116, + "full_recall_at_20": 0.6957, + "mean_recall_at_1000_tokens": 0.7319, + "mean_recall_at_4000_tokens": 0.8116, + "mean_recall_at_16000_tokens": 0.8116, + "mean_precision_at_10": 0.2379, + "zero_hit_rate": 0.087, + "median_tokens_delivered": 1051, + "median_latency_ms": 903.579, + "payload_kind": [ + "chunks" + ], + "unavailable": 1 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.4544, + "full_recall_at_5": 0.3396, + "mean_recall_at_10": 0.4969, + "full_recall_at_10": 0.3774, + "mean_recall_at_20": 0.4969, + "full_recall_at_20": 0.3774, + "mean_recall_at_1000_tokens": 0.4135, + "mean_recall_at_4000_tokens": 0.4969, + "mean_recall_at_16000_tokens": 0.4969, + "mean_precision_at_10": 0.1394, + "zero_hit_rate": 0.3774, + "median_tokens_delivered": 1227, + "median_latency_ms": 965.855, + "payload_kind": [ + "chunks" + ], + "unavailable": 4 + } + }, + "resources": { + "baseline_rss_mb": 78337.8, + "peak_rss_mb": 81637.5, + "delta_rss_mb": 3299.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.4583, + "full_recall_at_5": 0.3026, + "mean_recall_at_10": 0.5417, + "full_recall_at_10": 0.4211, + "mean_recall_at_20": 0.5789, + "full_recall_at_20": 0.4342, + "mean_recall_at_1000_tokens": 0.4342, + "mean_recall_at_4000_tokens": 0.568, + "mean_recall_at_16000_tokens": 0.5789, + "mean_precision_at_10": 0.1672, + "zero_hit_rate": 0.2632, + "median_tokens_delivered": 1670, + "median_latency_ms": 911.4735, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.2153, + "full_recall_at_5": 0.0833, + "mean_recall_at_10": 0.2639, + "full_recall_at_10": 0.1667, + "mean_recall_at_20": 0.3403, + "full_recall_at_20": 0.2083, + "mean_recall_at_1000_tokens": 0.1528, + "mean_recall_at_4000_tokens": 0.3264, + "mean_recall_at_16000_tokens": 0.3403, + "mean_precision_at_10": 0.1181, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 1691.5, + "median_latency_ms": 888.7285, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.5705, + "full_recall_at_5": 0.4038, + "mean_recall_at_10": 0.6699, + "full_recall_at_10": 0.5385, + "mean_recall_at_20": 0.6891, + "full_recall_at_20": 0.5385, + "mean_recall_at_1000_tokens": 0.5641, + "mean_recall_at_4000_tokens": 0.6795, + "mean_recall_at_16000_tokens": 0.6891, + "mean_precision_at_10": 0.1898, + "zero_hit_rate": 0.1538, + "median_tokens_delivered": 1668.5, + "median_latency_ms": 930.826, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.6196, + "full_recall_at_5": 0.4348, + "mean_recall_at_10": 0.7283, + "full_recall_at_10": 0.6087, + "mean_recall_at_20": 0.7283, + "full_recall_at_20": 0.6087, + "mean_recall_at_1000_tokens": 0.7065, + "mean_recall_at_4000_tokens": 0.7283, + "mean_recall_at_16000_tokens": 0.7283, + "mean_precision_at_10": 0.1979, + "zero_hit_rate": 0.1304, + "median_tokens_delivered": 1665, + "median_latency_ms": 906.547, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3884, + "full_recall_at_5": 0.2453, + "mean_recall_at_10": 0.4607, + "full_recall_at_10": 0.3396, + "mean_recall_at_20": 0.5142, + "full_recall_at_20": 0.3585, + "mean_recall_at_1000_tokens": 0.316, + "mean_recall_at_4000_tokens": 0.4984, + "mean_recall_at_16000_tokens": 0.5142, + "mean_precision_at_10": 0.1538, + "zero_hit_rate": 0.3208, + "median_tokens_delivered": 1674, + "median_latency_ms": 916.4, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 80183.9, + "peak_rss_mb": 81141.1, + "delta_rss_mb": 957.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 117551104 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.3783, + "full_recall_at_5": 0.2632, + "mean_recall_at_10": 0.5077, + "full_recall_at_10": 0.3553, + "mean_recall_at_20": 0.7368, + "full_recall_at_20": 0.6579, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0175, + "mean_recall_at_16000_tokens": 0.2884, + "mean_precision_at_10": 0.0934, + "zero_hit_rate": 0.1842, + "median_tokens_delivered": 88454.5, + "median_latency_ms": 21.344, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.2049, + "full_recall_at_5": 0.0833, + "mean_recall_at_10": 0.2882, + "full_recall_at_10": 0.125, + "mean_recall_at_20": 0.4653, + "full_recall_at_20": 0.3333, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0139, + "mean_recall_at_16000_tokens": 0.1215, + "mean_precision_at_10": 0.0625, + "zero_hit_rate": 0.4167, + "median_tokens_delivered": 88074, + "median_latency_ms": 21.407, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.4583, + "full_recall_at_5": 0.3462, + "mean_recall_at_10": 0.609, + "full_recall_at_10": 0.4615, + "mean_recall_at_20": 0.8622, + "full_recall_at_20": 0.8077, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0192, + "mean_recall_at_16000_tokens": 0.3654, + "mean_precision_at_10": 0.1077, + "zero_hit_rate": 0.0769, + "median_tokens_delivered": 89967, + "median_latency_ms": 21.303, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.4203, + "full_recall_at_5": 0.3478, + "mean_recall_at_10": 0.5507, + "full_recall_at_10": 0.3913, + "mean_recall_at_20": 0.7174, + "full_recall_at_20": 0.6087, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0435, + "mean_recall_at_16000_tokens": 0.3768, + "mean_precision_at_10": 0.0913, + "zero_hit_rate": 0.1739, + "median_tokens_delivered": 90059, + "median_latency_ms": 21.23, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3601, + "full_recall_at_5": 0.2264, + "mean_recall_at_10": 0.489, + "full_recall_at_10": 0.3396, + "mean_recall_at_20": 0.7453, + "full_recall_at_20": 0.6792, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0063, + "mean_recall_at_16000_tokens": 0.25, + "mean_precision_at_10": 0.0943, + "zero_hit_rate": 0.1887, + "median_tokens_delivered": 88304, + "median_latency_ms": 21.383, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 76914.8, + "peak_rss_mb": 76985.5, + "delta_rss_mb": 70.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.068, + "full_recall_at_5": 0.0132, + "mean_recall_at_10": 0.0921, + "full_recall_at_10": 0.0263, + "mean_recall_at_20": 0.1776, + "full_recall_at_20": 0.0526, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0263, + "mean_recall_at_16000_tokens": 0.1009, + "mean_precision_at_10": 0.0197, + "zero_hit_rate": 0.6842, + "median_tokens_delivered": 24792.5, + "median_latency_ms": 10.2995, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.0347, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0486, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1319, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0069, + "mean_recall_at_16000_tokens": 0.0764, + "mean_precision_at_10": 0.0167, + "zero_hit_rate": 0.7083, + "median_tokens_delivered": 25739, + "median_latency_ms": 10.2675, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.0833, + "full_recall_at_5": 0.0192, + "mean_recall_at_10": 0.1122, + "full_recall_at_10": 0.0385, + "mean_recall_at_20": 0.1987, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0353, + "mean_recall_at_16000_tokens": 0.1122, + "mean_precision_at_10": 0.0212, + "zero_hit_rate": 0.6731, + "median_tokens_delivered": 24792.5, + "median_latency_ms": 10.3385, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.0435, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0652, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1522, + "full_recall_at_20": 0.0435, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0217, + "mean_recall_at_16000_tokens": 0.0652, + "mean_precision_at_10": 0.013, + "zero_hit_rate": 0.7391, + "median_tokens_delivered": 24180, + "median_latency_ms": 10.102, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.0786, + "full_recall_at_5": 0.0189, + "mean_recall_at_10": 0.1038, + "full_recall_at_10": 0.0377, + "mean_recall_at_20": 0.1887, + "full_recall_at_20": 0.0566, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0283, + "mean_recall_at_16000_tokens": 0.1164, + "mean_precision_at_10": 0.0226, + "zero_hit_rate": 0.6604, + "median_tokens_delivered": 25977, + "median_latency_ms": 10.336, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 76979, + "peak_rss_mb": 77042.5, + "delta_rss_mb": 63.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "agent-default", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 59414, + "latency_ms": 79.903, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 71198, + "latency_ms": 64.149, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 76878, + "latency_ms": 71.881, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 57200, + "latency_ms": 67.159, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 25619, + "latency_ms": 57.064, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 67883, + "latency_ms": 53.576, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 42.847, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 73836, + "latency_ms": 77.798, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2441, + "latency_ms": 50.423, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 71759, + "latency_ms": 71.631, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 69187, + "latency_ms": 88.515, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 78746, + "latency_ms": 95.977, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 53332, + "latency_ms": 66.203, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 74381, + "latency_ms": 81.711, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 74795, + "latency_ms": 98.814, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 70497, + "latency_ms": 81.551, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13961, + "latency_ms": 56.675, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 79122, + "latency_ms": 49.615, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2143, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 71092, + "latency_ms": 66.76, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 66658, + "latency_ms": 57.885, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 20991, + "latency_ms": 76.389, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86943, + "latency_ms": 77.658, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 95744, + "latency_ms": 104.797, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 84785, + "latency_ms": 71.496, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67946, + "latency_ms": 53.697, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 67888, + "latency_ms": 60.568, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 47664, + "latency_ms": 68.793, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.8333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 61019, + "latency_ms": 52.649, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "gin.go", + "test_helpers.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 67859, + "latency_ms": 84.742, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 79884, + "latency_ms": 58.508, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 56517, + "latency_ms": 77.17, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 94416, + "latency_ms": 85.374, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 86766, + "latency_ms": 86.851, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 98524, + "latency_ms": 70.275, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 64060, + "latency_ms": 87.898, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 66751, + "latency_ms": 68.424, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 48437, + "latency_ms": 71.356, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 53447, + "latency_ms": 49.479, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 344, + "latency_ms": 50.195, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 76691, + "latency_ms": 72.592, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 63498, + "latency_ms": 73.044, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 79093, + "latency_ms": 70.947, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 60197, + "latency_ms": 50.534, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 59600, + "latency_ms": 62.999, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 62885, + "latency_ms": 54.646, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 72047, + "latency_ms": 73.408, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 76351, + "latency_ms": 92.836, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 66011, + "latency_ms": 90.734, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 21105, + "latency_ms": 52.431, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 58056, + "latency_ms": 52.241, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 90063, + "latency_ms": 78.401, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "recovery_test.go" + ], + "missed": [ + "errors.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 102004, + "latency_ms": 98.552, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 77237, + "latency_ms": 69.992, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 54210, + "latency_ms": 69.93, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 88221, + "latency_ms": 88.468, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 39288, + "latency_ms": 52.671, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47970, + "latency_ms": 54.6, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 58227, + "latency_ms": 69.9, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go" + ], + "missed": [ + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 92238, + "latency_ms": 78.978, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 103343, + "latency_ms": 54.166, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 64119, + "latency_ms": 61.637, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 47.229, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 65504, + "latency_ms": 96.215, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 87536, + "latency_ms": 77.593, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80910, + "latency_ms": 56.937, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 61055, + "latency_ms": 84.127, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 61558, + "latency_ms": 61.786, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 92311, + "latency_ms": 80.36, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 104156, + "latency_ms": 96.098, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 81488, + "latency_ms": 66.473, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 71433, + "latency_ms": 82.639, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 84500, + "latency_ms": 96.156, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 21802, + "latency_ms": 70.679, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 78640, + "latency_ms": 86.054, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 52456, + "latency_ms": 60.427, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 52554, + "latency_ms": 50.239, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 60149, + "latency_ms": 43.418, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 71198, + "latency_ms": 44.115, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 84027, + "latency_ms": 41.945, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 50548, + "latency_ms": 34.784, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 25619, + "latency_ms": 37.658, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 67883, + "latency_ms": 34.003, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 25.817, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 73836, + "latency_ms": 54.532, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2441, + "latency_ms": 34.464, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 75648, + "latency_ms": 44.053, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 69187, + "latency_ms": 42.651, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 78746, + "latency_ms": 53.28, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 53332, + "latency_ms": 27.128, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 74381, + "latency_ms": 55.608, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 86796, + "latency_ms": 68.041, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 70497, + "latency_ms": 54.021, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13961, + "latency_ms": 34.761, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 79122, + "latency_ms": 25.402, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2143, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 71092, + "latency_ms": 43.169, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 66658, + "latency_ms": 35.863, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 53169, + "latency_ms": 44.191, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86996, + "latency_ms": 44.422, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 98248, + "latency_ms": 75.962, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 84785, + "latency_ms": 48.917, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67946, + "latency_ms": 36.077, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 68660, + "latency_ms": 36.975, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 47664, + "latency_ms": 43.939, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.8333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 61019, + "latency_ms": 36.502, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "test_helpers.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 73530, + "latency_ms": 60.914, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 79884, + "latency_ms": 36.787, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 58532, + "latency_ms": 55.153, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 94416, + "latency_ms": 63.805, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 86766, + "latency_ms": 66.039, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 98524, + "latency_ms": 56.54, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 69644, + "latency_ms": 68.422, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 90825, + "latency_ms": 47.181, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 53828, + "latency_ms": 47.952, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 55487, + "latency_ms": 26.547, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 344, + "latency_ms": 25.679, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 76691, + "latency_ms": 52.518, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode_test.go" + ], + "missed": [ + "mode.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 65844, + "latency_ms": 46.042, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 79093, + "latency_ms": 54.475, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62345, + "latency_ms": 23.772, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 59600, + "latency_ms": 46.707, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 62885, + "latency_ms": 34.846, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 73065, + "latency_ms": 45.055, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 84515, + "latency_ms": 61.637, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 68823, + "latency_ms": 64.541, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 64033, + "latency_ms": 25.805, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 58056, + "latency_ms": 26.617, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93773, + "latency_ms": 55.829, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "recovery_test.go" + ], + "missed": [ + "errors.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 102004, + "latency_ms": 85.152, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 77237, + "latency_ms": 43.889, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 58472, + "latency_ms": 46.259, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93890, + "latency_ms": 63.052, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 39288, + "latency_ms": 35.157, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47970, + "latency_ms": 36.115, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 59839, + "latency_ms": 42.998, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go" + ], + "missed": [ + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 92238, + "latency_ms": 58.627, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 103343, + "latency_ms": 35.098, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 69750, + "latency_ms": 32.06, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 23.784, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 87717, + "latency_ms": 69.443, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 87368, + "latency_ms": 50.235, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80910, + "latency_ms": 32.251, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 61055, + "latency_ms": 62.49, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 61558, + "latency_ms": 43.862, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 92311, + "latency_ms": 61.84, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 104156, + "latency_ms": 68.25, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82299, + "latency_ms": 40.075, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 71447, + "latency_ms": 59.985, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 84500, + "latency_ms": 74.521, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 70865, + "latency_ms": 44.778, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 78640, + "latency_ms": 66.308, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 56727, + "latency_ms": 32.339, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 64337, + "latency_ms": 25.263, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4742, + "latency_ms": 25322.083, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 518, + "latency_ms": 10764.447, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 914, + "latency_ms": 7130.394, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1339, + "latency_ms": 2124.872, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2062, + "latency_ms": 1891.216, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1027, + "latency_ms": 945.449, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 483, + "latency_ms": 1717.939, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1509, + "latency_ms": 15123.175, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1075.095, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 897, + "latency_ms": 4256.821, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1034, + "latency_ms": 890.219, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1188, + "latency_ms": 9130.49, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1368, + "latency_ms": 1223.566, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 722, + "latency_ms": 934.753, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1104, + "latency_ms": 859.803, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 974, + "latency_ms": 1007.918, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1227, + "latency_ms": 788.487, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 693, + "latency_ms": 1424.016, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.3, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3585, + "latency_ms": 831.256, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 976, + "latency_ms": 854.361, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 766, + "latency_ms": 841.266, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1313, + "latency_ms": 797.444, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2780, + "latency_ms": 1099.659, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1180, + "latency_ms": 965.855, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1275, + "latency_ms": 887.46, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "routes_test.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 515, + "latency_ms": 1130.131, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 797, + "latency_ms": 1025.613, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "binding/default_validator.go", + "gin.go" + ], + "missed": [ + "context_test.go", + "errors.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 2517, + "latency_ms": 820.584, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "test_helpers.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 1309, + "latency_ms": 845.233, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2658, + "latency_ms": 1204.528, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1045, + "latency_ms": 903.579, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 740, + "latency_ms": 871.719, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1841, + "latency_ms": 1090.465, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1078, + "latency_ms": 874.37, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 772, + "latency_ms": 874.002, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1323, + "latency_ms": 2245.537, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1390.749, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 501, + "latency_ms": 1968.616, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2857, + "recall_at_20": 0.5, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1051, + "latency_ms": 965.034, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 831.776, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.3, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1083, + "latency_ms": 886.594, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3122, + "latency_ms": 1179.317, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 814.916, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 478, + "latency_ms": 816.356, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 800, + "latency_ms": 819.699, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 758, + "latency_ms": 837.859, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 982, + "latency_ms": 927.401, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2592, + "latency_ms": 912.964, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 643, + "latency_ms": 832.95, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1261, + "latency_ms": 852.161, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1574, + "latency_ms": 1146.278, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "errors.go" + ], + "missed": [ + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2857, + "recall_at_20": 0.5, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1750, + "latency_ms": 993.112, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4044, + "latency_ms": 843.616, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 649, + "latency_ms": 879.176, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4069, + "latency_ms": 1002.089, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3216, + "latency_ms": 871.182, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1358, + "latency_ms": 873.539, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1362, + "latency_ms": 923.837, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go" + ], + "missed": [ + "errors_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 765, + "latency_ms": 833.56, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 897, + "latency_ms": 905.355, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 546, + "latency_ms": 884.154, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1387, + "latency_ms": 1056.028, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2075, + "latency_ms": 994.099, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2284, + "latency_ms": 1004.194, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1221, + "latency_ms": 973.703, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 842.6, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1967, + "latency_ms": 832.381, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 750, + "latency_ms": 972.052, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1771, + "latency_ms": 939.853, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2505, + "latency_ms": 1010.787, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2502, + "latency_ms": 859.993, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1921, + "latency_ms": 1205.904, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/protobuf.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 925, + "latency_ms": 854.482, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1237, + "latency_ms": 1032.483, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1338, + "latency_ms": 1073.857, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1295, + "latency_ms": 1073.049, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [ + "logger_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 5, + "tokens_delivered": 6018, + "latency_ms": 1149.043, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 1674, + "latency_ms": 1156.341, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1663, + "latency_ms": 1127.005, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1665, + "latency_ms": 1016.789, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 6022, + "latency_ms": 1012.918, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 7862, + "latency_ms": 898.613, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4075, + "latency_ms": 995.277, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1672, + "latency_ms": 1115.977, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7, + "latency_ms": 968.478, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 222, + "latency_ms": 1083.757, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 667, + "latency_ms": 991.604, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 191, + "latency_ms": 1291.424, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 81, + "latency_ms": 1350.962, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1668, + "latency_ms": 1518.012, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1659, + "latency_ms": 857.242, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4145, + "latency_ms": 794.373, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8733, + "latency_ms": 741.315, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 1673, + "latency_ms": 815.265, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go", + "tree_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 160, + "latency_ms": 788.396, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1677, + "latency_ms": 756.969, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4283, + "latency_ms": 781.717, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 495, + "latency_ms": 809.642, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1662, + "latency_ms": 923.277, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 1676, + "latency_ms": 917.048, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2269, + "latency_ms": 823.256, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1663, + "latency_ms": 782.638, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2061, + "latency_ms": 792.898, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.5, + "precision_at_20": 0.3, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1666, + "latency_ms": 794.125, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.1579, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2706, + "latency_ms": 844.553, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 484, + "latency_ms": 930.394, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1663, + "latency_ms": 952.567, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1670, + "latency_ms": 977.624, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1670, + "latency_ms": 1049.046, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1673, + "latency_ms": 1013.982, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2333, + "latency_ms": 873.742, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8411, + "latency_ms": 960.13, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1665, + "latency_ms": 879.189, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1655, + "latency_ms": 816.057, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 1, + "recall_at_10": 0.25, + "precision_at_10": 1, + "recall_at_20": 0.25, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 129, + "latency_ms": 906.547, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1661, + "latency_ms": 884.83, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1669, + "latency_ms": 893.379, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 145, + "latency_ms": 836.247, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 19, + "tokens_delivered": 1674, + "latency_ms": 855.677, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5798, + "latency_ms": 855.386, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/validate_test.go" + ], + "missed": [ + "binding/default_validator.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2608, + "latency_ms": 905.941, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1667, + "latency_ms": 885.312, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1138, + "latency_ms": 931.258, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1669, + "latency_ms": 905.866, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1038, + "latency_ms": 880.78, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 210, + "latency_ms": 916.4, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1706, + "latency_ms": 1088.582, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 4, + "tokens_delivered": 3336, + "latency_ms": 1100.476, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1663, + "latency_ms": 1103.652, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1663, + "latency_ms": 941.68, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 6602, + "latency_ms": 1210.955, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4366, + "latency_ms": 997.855, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1677, + "latency_ms": 984.752, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 1667, + "latency_ms": 994.103, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1665, + "latency_ms": 1053.01, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6296, + "latency_ms": 1075.676, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1665, + "latency_ms": 1049.441, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1667, + "latency_ms": 878.463, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1682, + "latency_ms": 1034.125, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 3782, + "latency_ms": 1028.837, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 193, + "latency_ms": 896.677, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3970, + "latency_ms": 781.71, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1687, + "latency_ms": 782.941, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1670, + "latency_ms": 933.32, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 5273, + "latency_ms": 934.744, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 6577, + "latency_ms": 818.779, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1667, + "latency_ms": 846.643, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5820, + "latency_ms": 873.611, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/protobuf.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 7343, + "latency_ms": 812.766, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7724, + "latency_ms": 820.129, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug_test.go" + ], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 20, + "tokens_delivered": 1680, + "latency_ms": 740.636, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1665, + "latency_ms": 791.738, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 69810, + "latency_ms": 26.636, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 84576, + "latency_ms": 21.43, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87509, + "latency_ms": 20.97, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 82399, + "latency_ms": 21.183, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 83992, + "latency_ms": 20.396, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 72743, + "latency_ms": 20.329, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 88246, + "latency_ms": 21.029, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105412, + "latency_ms": 22.158, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 84449, + "latency_ms": 21.521, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 95317, + "latency_ms": 22.044, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 69979, + "latency_ms": 21.788, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 113276, + "latency_ms": 20.521, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 108509, + "latency_ms": 20.783, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 84446, + "latency_ms": 22.427, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 90304, + "latency_ms": 20.472, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 89875, + "latency_ms": 20.597, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70138, + "latency_ms": 20.668, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93950, + "latency_ms": 21.419, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 87314, + "latency_ms": 21.154, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82768, + "latency_ms": 20.465, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 88101, + "latency_ms": 20.8, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 87276, + "latency_ms": 21.906, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 117055, + "latency_ms": 20.836, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 113658, + "latency_ms": 20.993, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 87849, + "latency_ms": 20.909, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 89097, + "latency_ms": 22.053, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93972, + "latency_ms": 22.646, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 88010, + "latency_ms": 21.763, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "test_helpers.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 88138, + "latency_ms": 21.23, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 103341, + "latency_ms": 21.961, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 115951, + "latency_ms": 22.703, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 106775, + "latency_ms": 22.678, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 94928, + "latency_ms": 21.492, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 110221, + "latency_ms": 20.729, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87531, + "latency_ms": 21.787, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 102446, + "latency_ms": 21.854, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 83359, + "latency_ms": 20.675, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 66540, + "latency_ms": 21.434, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 101365, + "latency_ms": 21.694, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 88522, + "latency_ms": 21.941, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 92380, + "latency_ms": 21.174, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 82724, + "latency_ms": 21.274, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 88010, + "latency_ms": 22.098, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 84577, + "latency_ms": 20.945, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 88304, + "latency_ms": 23.14, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 88116, + "latency_ms": 22.418, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 98912, + "latency_ms": 21.332, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 86657, + "latency_ms": 20.84, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 90329, + "latency_ms": 22.024, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88014, + "latency_ms": 21.803, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 112651, + "latency_ms": 21.431, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "recovery_test.go" + ], + "missed": [ + "errors.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 115859, + "latency_ms": 23.44, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88387, + "latency_ms": 21.628, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 82791, + "latency_ms": 21.356, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 111258, + "latency_ms": 22.073, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88075, + "latency_ms": 20.867, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86659, + "latency_ms": 20.79, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 88259, + "latency_ms": 21.097, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 101364, + "latency_ms": 21.891, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 108196, + "latency_ms": 21.172, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 90059, + "latency_ms": 21.091, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [ + "recovery.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 72362, + "latency_ms": 21.855, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 105612, + "latency_ms": 21.119, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 106371, + "latency_ms": 20.382, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 92820, + "latency_ms": 21.054, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 83426, + "latency_ms": 22.191, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 84119, + "latency_ms": 21.16, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 113276, + "latency_ms": 21.065, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 113946, + "latency_ms": 21.401, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93998, + "latency_ms": 21.427, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 98912, + "latency_ms": 20.816, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 98262, + "latency_ms": 20.599, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "binding/protobuf.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 82704, + "latency_ms": 21.697, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 99455, + "latency_ms": 21.383, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 68234, + "latency_ms": 20.355, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 90299, + "latency_ms": 20.532, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go", + "logger_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 44600, + "latency_ms": 12.215, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12761, + "latency_ms": 11.384, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26285, + "latency_ms": 10.448, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24674, + "latency_ms": 11.295, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21162, + "latency_ms": 9.758, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 27850, + "latency_ms": 10.858, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28556, + "latency_ms": 11.354, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18035, + "latency_ms": 10.295, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19033, + "latency_ms": 10.806, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 37122, + "latency_ms": 10.489, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30316, + "latency_ms": 10.622, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 33394, + "latency_ms": 11.531, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26404, + "latency_ms": 10.512, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26292, + "latency_ms": 10.267, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22473, + "latency_ms": 10.876, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 15956, + "latency_ms": 11.564, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19388, + "latency_ms": 9.993, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17445, + "latency_ms": 11.534, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go" + ], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 27773, + "latency_ms": 10.382, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14497, + "latency_ms": 9.986, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30553, + "latency_ms": 10.717, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 37728, + "latency_ms": 10.642, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 34527, + "latency_ms": 10.054, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19547, + "latency_ms": 10.911, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12483, + "latency_ms": 9.923, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 22479, + "latency_ms": 10.683, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14254, + "latency_ms": 10.982, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 35496, + "latency_ms": 10.76, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [], + "missed": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23550, + "latency_ms": 10.231, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18818, + "latency_ms": 10.454, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 48958, + "latency_ms": 9.647, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53000, + "latency_ms": 11.565, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 28956, + "latency_ms": 10.057, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21545, + "latency_ms": 9.851, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 34999, + "latency_ms": 9.84, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 26477, + "latency_ms": 9.671, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20935, + "latency_ms": 9.856, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20327, + "latency_ms": 9.309, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21566, + "latency_ms": 10.012, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25538, + "latency_ms": 10.258, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24170, + "latency_ms": 9.58, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23086, + "latency_ms": 9.845, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20136, + "latency_ms": 9.099, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8753, + "latency_ms": 10.04, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35582, + "latency_ms": 9.484, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 27255, + "latency_ms": 9.704, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30122, + "latency_ms": 9.83, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [], + "missed": [ + "gin.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19879, + "latency_ms": 9.394, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33426, + "latency_ms": 9.58, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16642, + "latency_ms": 9.553, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27618, + "latency_ms": 9.903, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "errors.go", + "routergroup.go" + ], + "missed": [ + "context.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 39163, + "latency_ms": 10.304, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 16118, + "latency_ms": 9.495, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23279, + "latency_ms": 10.658, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 36634, + "latency_ms": 11.826, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 40307, + "latency_ms": 10.513, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23860, + "latency_ms": 10.916, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37903, + "latency_ms": 9.862, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49541, + "latency_ms": 10.31, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44167, + "latency_ms": 10.336, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13012, + "latency_ms": 9.687, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28129, + "latency_ms": 10.617, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19778, + "latency_ms": 10.102, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 27314, + "latency_ms": 9.849, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22773, + "latency_ms": 9.895, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 28023, + "latency_ms": 9.389, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24288, + "latency_ms": 9.674, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26606, + "latency_ms": 9.294, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21083, + "latency_ms": 10.482, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8705, + "latency_ms": 10.869, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55452, + "latency_ms": 10.638, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 25977, + "latency_ms": 10.864, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/protobuf.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 24180, + "latency_ms": 11.252, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16819, + "latency_ms": 10.584, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 15488, + "latency_ms": 10.172, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24911, + "latency_ms": 10.542, + "indexed_revision_matches": true + } + ], + "tier": "small", + "tier_code_files": 99, + "tier_provenance": "Stamped after the fact from a measured file count; the run predates tier stamping. Tier is a property of the repository, not of the run." +} diff --git a/benchmarks/context-retrieval/results/score-pub-got.json b/benchmarks/context-retrieval/results/score-pub-got.json new file mode 100644 index 00000000..7abb5105 --- /dev/null +++ b/benchmarks/context-retrieval/results/score-pub-got.json @@ -0,0 +1,18417 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "got", + "head": "e3924aa1e53a6ca3eb93a43618ce532442a89b40" + }, + "corpus_counts": { + "cases": 108, + "path_leak": 54, + "baseline_blind": 54, + "multi_file": 82 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "providers": [ + { + "provider_id": "agent-default", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.4801, + "full_recall_at_5": 0.3056, + "mean_recall_at_10": 0.6475, + "full_recall_at_10": 0.4722, + "mean_recall_at_20": 0.8486, + "full_recall_at_20": 0.75, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.159, + "mean_recall_at_16000_tokens": 0.4474, + "mean_precision_at_10": 0.1329, + "zero_hit_rate": 0.0741, + "median_tokens_delivered": 59966.5, + "median_latency_ms": 52.1365, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 57, + "mean_recall_at_5": 0.2035, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.3322, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.7132, + "full_recall_at_20": 0.5263, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0643, + "mean_recall_at_16000_tokens": 0.1942, + "mean_precision_at_10": 0.086, + "zero_hit_rate": 0.1404, + "median_tokens_delivered": 60939, + "median_latency_ms": 51.84, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 51, + "mean_recall_at_5": 0.7892, + "full_recall_at_5": 0.6471, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2647, + "mean_recall_at_16000_tokens": 0.7304, + "mean_precision_at_10": 0.1853, + "zero_hit_rate": 0, + "median_tokens_delivered": 58677, + "median_latency_ms": 52.753, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.6716, + "full_recall_at_5": 0.463, + "mean_recall_at_10": 0.7765, + "full_recall_at_10": 0.6296, + "mean_recall_at_20": 0.9179, + "full_recall_at_20": 0.8148, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.2901, + "mean_recall_at_16000_tokens": 0.6278, + "mean_precision_at_10": 0.1556, + "zero_hit_rate": 0, + "median_tokens_delivered": 60910, + "median_latency_ms": 54.7415, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.2886, + "full_recall_at_5": 0.1481, + "mean_recall_at_10": 0.5185, + "full_recall_at_10": 0.3148, + "mean_recall_at_20": 0.7793, + "full_recall_at_20": 0.6852, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0278, + "mean_recall_at_16000_tokens": 0.267, + "mean_precision_at_10": 0.1102, + "zero_hit_rate": 0.1481, + "median_tokens_delivered": 59722, + "median_latency_ms": 50.6835, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 76998.3, + "peak_rss_mb": 77094.2, + "delta_rss_mb": 95.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "keyword-search", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.4119, + "full_recall_at_5": 0.1944, + "mean_recall_at_10": 0.5809, + "full_recall_at_10": 0.3611, + "mean_recall_at_20": 0.81, + "full_recall_at_20": 0.6944, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1065, + "mean_recall_at_16000_tokens": 0.391, + "mean_precision_at_10": 0.1208, + "zero_hit_rate": 0.0926, + "median_tokens_delivered": 62238.5, + "median_latency_ms": 32.2075, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 57, + "mean_recall_at_5": 0.2117, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.352, + "full_recall_at_10": 0.0351, + "mean_recall_at_20": 0.6839, + "full_recall_at_20": 0.5088, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0263, + "mean_recall_at_16000_tokens": 0.1927, + "mean_precision_at_10": 0.0895, + "zero_hit_rate": 0.1754, + "median_tokens_delivered": 63960, + "median_latency_ms": 31.822, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 51, + "mean_recall_at_5": 0.6356, + "full_recall_at_5": 0.4118, + "mean_recall_at_10": 0.8366, + "full_recall_at_10": 0.7255, + "mean_recall_at_20": 0.951, + "full_recall_at_20": 0.902, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1961, + "mean_recall_at_16000_tokens": 0.6127, + "mean_precision_at_10": 0.1559, + "zero_hit_rate": 0, + "median_tokens_delivered": 60881, + "median_latency_ms": 34.19, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.4488, + "full_recall_at_5": 0.2222, + "mean_recall_at_10": 0.6093, + "full_recall_at_10": 0.3889, + "mean_recall_at_20": 0.8346, + "full_recall_at_20": 0.6852, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1574, + "mean_recall_at_16000_tokens": 0.4333, + "mean_precision_at_10": 0.1241, + "zero_hit_rate": 0.037, + "median_tokens_delivered": 63529.5, + "median_latency_ms": 33.195, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.375, + "full_recall_at_5": 0.1667, + "mean_recall_at_10": 0.5525, + "full_recall_at_10": 0.3333, + "mean_recall_at_20": 0.7855, + "full_recall_at_20": 0.7037, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0556, + "mean_recall_at_16000_tokens": 0.3488, + "mean_precision_at_10": 0.1176, + "zero_hit_rate": 0.1481, + "median_tokens_delivered": 61941.5, + "median_latency_ms": 31.8165, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 77083.7, + "peak_rss_mb": 77147.5, + "delta_rss_mb": 63.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codesearch", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.3596, + "full_recall_at_5": 0.25, + "mean_recall_at_10": 0.4182, + "full_recall_at_10": 0.3056, + "mean_recall_at_20": 0.4228, + "full_recall_at_20": 0.3056, + "mean_recall_at_1000_tokens": 0.3148, + "mean_recall_at_4000_tokens": 0.4228, + "mean_recall_at_16000_tokens": 0.4228, + "mean_precision_at_10": 0.145, + "zero_hit_rate": 0.463, + "median_tokens_delivered": 850, + "median_latency_ms": 986.179, + "payload_kind": [ + "chunks" + ], + "unavailable": 41 + }, + "baseline_missed": { + "cases": 57, + "mean_recall_at_5": 0.2237, + "full_recall_at_5": 0.1228, + "mean_recall_at_10": 0.2939, + "full_recall_at_10": 0.1579, + "mean_recall_at_20": 0.3026, + "full_recall_at_20": 0.1579, + "mean_recall_at_1000_tokens": 0.2149, + "mean_recall_at_4000_tokens": 0.3026, + "mean_recall_at_16000_tokens": 0.3026, + "mean_precision_at_10": 0.0927, + "zero_hit_rate": 0.5614, + "median_tokens_delivered": 403, + "median_latency_ms": 963.19, + "payload_kind": [ + "chunks" + ], + "unavailable": 27 + }, + "baseline_solved": { + "cases": 51, + "mean_recall_at_5": 0.5114, + "full_recall_at_5": 0.3922, + "mean_recall_at_10": 0.5572, + "full_recall_at_10": 0.4706, + "mean_recall_at_20": 0.5572, + "full_recall_at_20": 0.4706, + "mean_recall_at_1000_tokens": 0.4265, + "mean_recall_at_4000_tokens": 0.5572, + "mean_recall_at_16000_tokens": 0.5572, + "mean_precision_at_10": 0.2034, + "zero_hit_rate": 0.3529, + "median_tokens_delivered": 1029, + "median_latency_ms": 1059.737, + "payload_kind": [ + "chunks" + ], + "unavailable": 14 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.4537, + "full_recall_at_5": 0.2963, + "mean_recall_at_10": 0.4907, + "full_recall_at_10": 0.3519, + "mean_recall_at_20": 0.5, + "full_recall_at_20": 0.3519, + "mean_recall_at_1000_tokens": 0.3611, + "mean_recall_at_4000_tokens": 0.5, + "mean_recall_at_16000_tokens": 0.5, + "mean_precision_at_10": 0.1991, + "zero_hit_rate": 0.3704, + "median_tokens_delivered": 957, + "median_latency_ms": 987.7255, + "payload_kind": [ + "chunks" + ], + "unavailable": 17 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.2654, + "full_recall_at_5": 0.2037, + "mean_recall_at_10": 0.3457, + "full_recall_at_10": 0.2593, + "mean_recall_at_20": 0.3457, + "full_recall_at_20": 0.2593, + "mean_recall_at_1000_tokens": 0.2685, + "mean_recall_at_4000_tokens": 0.3457, + "mean_recall_at_16000_tokens": 0.3457, + "mean_precision_at_10": 0.0908, + "zero_hit_rate": 0.5556, + "median_tokens_delivered": 785, + "median_latency_ms": 980.6065, + "payload_kind": [ + "chunks" + ], + "unavailable": 24 + } + }, + "resources": { + "baseline_rss_mb": 76915.1, + "peak_rss_mb": 105468.8, + "delta_rss_mb": 28553.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.284, + "full_recall_at_5": 0.1204, + "mean_recall_at_10": 0.3957, + "full_recall_at_10": 0.213, + "mean_recall_at_20": 0.4577, + "full_recall_at_20": 0.2963, + "mean_recall_at_1000_tokens": 0.2887, + "mean_recall_at_4000_tokens": 0.4346, + "mean_recall_at_16000_tokens": 0.4577, + "mean_precision_at_10": 0.1039, + "zero_hit_rate": 0.3981, + "median_tokens_delivered": 1675, + "median_latency_ms": 986.3215, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 57, + "mean_recall_at_5": 0.2485, + "full_recall_at_5": 0.0877, + "mean_recall_at_10": 0.3637, + "full_recall_at_10": 0.193, + "mean_recall_at_20": 0.4374, + "full_recall_at_20": 0.2807, + "mean_recall_at_1000_tokens": 0.2518, + "mean_recall_at_4000_tokens": 0.4023, + "mean_recall_at_16000_tokens": 0.4374, + "mean_precision_at_10": 0.0988, + "zero_hit_rate": 0.4386, + "median_tokens_delivered": 1678, + "median_latency_ms": 985.927, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 51, + "mean_recall_at_5": 0.3235, + "full_recall_at_5": 0.1569, + "mean_recall_at_10": 0.4314, + "full_recall_at_10": 0.2353, + "mean_recall_at_20": 0.4804, + "full_recall_at_20": 0.3137, + "mean_recall_at_1000_tokens": 0.3301, + "mean_recall_at_4000_tokens": 0.4706, + "mean_recall_at_16000_tokens": 0.4804, + "mean_precision_at_10": 0.1096, + "zero_hit_rate": 0.3529, + "median_tokens_delivered": 1671, + "median_latency_ms": 991.965, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.2253, + "full_recall_at_5": 0.0741, + "mean_recall_at_10": 0.3407, + "full_recall_at_10": 0.1481, + "mean_recall_at_20": 0.4123, + "full_recall_at_20": 0.2593, + "mean_recall_at_1000_tokens": 0.292, + "mean_recall_at_4000_tokens": 0.4123, + "mean_recall_at_16000_tokens": 0.4123, + "mean_precision_at_10": 0.0852, + "zero_hit_rate": 0.4444, + "median_tokens_delivered": 1674, + "median_latency_ms": 989.1725, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3426, + "full_recall_at_5": 0.1667, + "mean_recall_at_10": 0.4506, + "full_recall_at_10": 0.2778, + "mean_recall_at_20": 0.5031, + "full_recall_at_20": 0.3333, + "mean_recall_at_1000_tokens": 0.2855, + "mean_recall_at_4000_tokens": 0.4568, + "mean_recall_at_16000_tokens": 0.5031, + "mean_precision_at_10": 0.1225, + "zero_hit_rate": 0.3519, + "median_tokens_delivered": 1677, + "median_latency_ms": 985.532, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 75351.3, + "peak_rss_mb": 80161.8, + "delta_rss_mb": 4810.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 84590592 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.4293, + "full_recall_at_5": 0.1852, + "mean_recall_at_10": 0.5823, + "full_recall_at_10": 0.3426, + "mean_recall_at_20": 0.7793, + "full_recall_at_20": 0.6019, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0278, + "mean_recall_at_16000_tokens": 0.3735, + "mean_precision_at_10": 0.1157, + "zero_hit_rate": 0.0741, + "median_tokens_delivered": 76435, + "median_latency_ms": 28.6395, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 57, + "mean_recall_at_5": 0.4289, + "full_recall_at_5": 0.193, + "mean_recall_at_10": 0.6091, + "full_recall_at_10": 0.4035, + "mean_recall_at_20": 0.7792, + "full_recall_at_20": 0.614, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0351, + "mean_recall_at_16000_tokens": 0.3845, + "mean_precision_at_10": 0.1281, + "zero_hit_rate": 0.0877, + "median_tokens_delivered": 76435, + "median_latency_ms": 28.506, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 51, + "mean_recall_at_5": 0.4297, + "full_recall_at_5": 0.1765, + "mean_recall_at_10": 0.5523, + "full_recall_at_10": 0.2745, + "mean_recall_at_20": 0.7794, + "full_recall_at_20": 0.5882, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0196, + "mean_recall_at_16000_tokens": 0.3611, + "mean_precision_at_10": 0.102, + "zero_hit_rate": 0.0588, + "median_tokens_delivered": 76435, + "median_latency_ms": 28.818, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.3401, + "full_recall_at_5": 0.0926, + "mean_recall_at_10": 0.4963, + "full_recall_at_10": 0.2037, + "mean_recall_at_20": 0.7161, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0093, + "mean_recall_at_16000_tokens": 0.2778, + "mean_precision_at_10": 0.1056, + "zero_hit_rate": 0.0926, + "median_tokens_delivered": 76984, + "median_latency_ms": 28.8405, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.5185, + "full_recall_at_5": 0.2778, + "mean_recall_at_10": 0.6682, + "full_recall_at_10": 0.4815, + "mean_recall_at_20": 0.8426, + "full_recall_at_20": 0.7037, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0463, + "mean_recall_at_16000_tokens": 0.4691, + "mean_precision_at_10": 0.1259, + "zero_hit_rate": 0.0556, + "median_tokens_delivered": 75083.5, + "median_latency_ms": 28.4765, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 78487.2, + "peak_rss_mb": 78501.8, + "delta_rss_mb": 14.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 108, + "mean_recall_at_5": 0.0968, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.1927, + "full_recall_at_10": 0.0741, + "mean_recall_at_20": 0.359, + "full_recall_at_20": 0.1481, + "mean_recall_at_1000_tokens": 0.0031, + "mean_recall_at_4000_tokens": 0.0296, + "mean_recall_at_16000_tokens": 0.1954, + "mean_precision_at_10": 0.0361, + "zero_hit_rate": 0.4167, + "median_tokens_delivered": 29954, + "median_latency_ms": 12.686, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 57, + "mean_recall_at_5": 0.0883, + "full_recall_at_5": 0.0526, + "mean_recall_at_10": 0.1781, + "full_recall_at_10": 0.0702, + "mean_recall_at_20": 0.357, + "full_recall_at_20": 0.1228, + "mean_recall_at_1000_tokens": 0.0058, + "mean_recall_at_4000_tokens": 0.0561, + "mean_recall_at_16000_tokens": 0.1845, + "mean_precision_at_10": 0.0351, + "zero_hit_rate": 0.3684, + "median_tokens_delivered": 30113, + "median_latency_ms": 12.567, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 51, + "mean_recall_at_5": 0.1062, + "full_recall_at_5": 0.0196, + "mean_recall_at_10": 0.2092, + "full_recall_at_10": 0.0784, + "mean_recall_at_20": 0.3611, + "full_recall_at_20": 0.1765, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0.2075, + "mean_precision_at_10": 0.0373, + "zero_hit_rate": 0.4706, + "median_tokens_delivered": 29680, + "median_latency_ms": 12.757, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 54, + "mean_recall_at_5": 0.1302, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.242, + "full_recall_at_10": 0.1111, + "mean_recall_at_20": 0.3753, + "full_recall_at_20": 0.1667, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.016, + "mean_recall_at_16000_tokens": 0.2519, + "mean_precision_at_10": 0.0426, + "zero_hit_rate": 0.4074, + "median_tokens_delivered": 29934, + "median_latency_ms": 12.555, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 54, + "mean_recall_at_5": 0.0633, + "full_recall_at_5": 0.037, + "mean_recall_at_10": 0.1435, + "full_recall_at_10": 0.037, + "mean_recall_at_20": 0.3426, + "full_recall_at_20": 0.1296, + "mean_recall_at_1000_tokens": 0.0062, + "mean_recall_at_4000_tokens": 0.0432, + "mean_recall_at_16000_tokens": 0.1389, + "mean_precision_at_10": 0.0296, + "zero_hit_rate": 0.4259, + "median_tokens_delivered": 29954, + "median_latency_ms": 12.865, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 78518.4, + "peak_rss_mb": 78712.5, + "delta_rss_mb": 194.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "agent-default", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 52061, + "latency_ms": 66.338, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 60535, + "latency_ms": 55.458, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 76099, + "latency_ms": 49.12, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53804, + "latency_ms": 39.784, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 68497, + "latency_ms": 53.298, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 155510, + "latency_ms": 74.779, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 52600, + "latency_ms": 63.683, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 53303, + "latency_ms": 49.958, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 87223, + "latency_ms": 71.7, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 45358, + "latency_ms": 45.433, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 102374, + "latency_ms": 50.906, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 46089, + "latency_ms": 44.586, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/normalize-arguments.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 36976, + "latency_ms": 44.427, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 47403, + "latency_ms": 38.536, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 64066, + "latency_ms": 66.596, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 73125, + "latency_ms": 63.922, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 70138, + "latency_ms": 64.921, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 65691, + "latency_ms": 64.67, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 53642, + "latency_ms": 66.762, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 45399, + "latency_ms": 50.461, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 73783, + "latency_ms": 47.291, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 40247, + "latency_ms": 57.029, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 80402, + "latency_ms": 63.418, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 43167, + "latency_ms": 41.515, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 75593, + "latency_ms": 46.99, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 46583, + "latency_ms": 42.138, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 60881, + "latency_ms": 51.431, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 41278, + "latency_ms": 46.432, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 55299, + "latency_ms": 53.482, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 59733, + "latency_ms": 45.827, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 46609, + "latency_ms": 41.171, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 59711, + "latency_ms": 54.25, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 37682, + "latency_ms": 43.777, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 96333, + "latency_ms": 51.978, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62246, + "latency_ms": 32.479, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 91683, + "latency_ms": 63.974, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 80333, + "latency_ms": 60.541, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 41266, + "latency_ms": 61.709, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [ + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 4, + "tokens_delivered": 23953, + "latency_ms": 45.55, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 143218, + "latency_ms": 57.163, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 62776, + "latency_ms": 51.211, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 79989, + "latency_ms": 77.707, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 89231, + "latency_ms": 46.477, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 28709, + "latency_ms": 32.637, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 31781, + "latency_ms": 55.38, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48396, + "latency_ms": 48.305, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 78960, + "latency_ms": 63.658, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 86165, + "latency_ms": 70.589, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 75355, + "latency_ms": 67.021, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 59030, + "latency_ms": 40.452, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 80690, + "latency_ms": 58.364, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 6, + "tokens_delivered": 46560, + "latency_ms": 64.423, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 137075, + "latency_ms": 59.179, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 59276, + "latency_ms": 51.852, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 75580, + "latency_ms": 63.972, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 58669, + "latency_ms": 54.377, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 44011, + "latency_ms": 52.753, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 43641, + "latency_ms": 37.974, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 30837, + "latency_ms": 42.8, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 63274, + "latency_ms": 49.617, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 49650, + "latency_ms": 44.831, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 44256, + "latency_ms": 48.402, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 36292, + "latency_ms": 52.295, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 72027, + "latency_ms": 49.401, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 42752, + "latency_ms": 59.466, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 83669, + "latency_ms": 43.141, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [ + "test/progress.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 42213, + "latency_ms": 45.201, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 42004, + "latency_ms": 39.67, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 49037, + "latency_ms": 59.309, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 60200, + "latency_ms": 49.343, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 69811, + "latency_ms": 57.248, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 77408, + "latency_ms": 59.362, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 72280, + "latency_ms": 59.545, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 62026, + "latency_ms": 55.106, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 32327, + "latency_ms": 44.111, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 142902, + "latency_ms": 64.838, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48778, + "latency_ms": 43.112, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 143714, + "latency_ms": 51.522, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 48415, + "latency_ms": 47.503, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28401, + "latency_ms": 33.095, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 85685, + "latency_ms": 51.287, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34044, + "latency_ms": 50.285, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 44953, + "latency_ms": 50.264, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 53717, + "latency_ms": 65.094, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 44547, + "latency_ms": 54.37, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 65288, + "latency_ms": 51.84, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 71129, + "latency_ms": 67.374, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 76417, + "latency_ms": 72.728, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 61652, + "latency_ms": 42.078, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 46441, + "latency_ms": 52.993, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 46586, + "latency_ms": 68.501, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 66791, + "latency_ms": 70.922, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 62475, + "latency_ms": 69.031, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33797, + "latency_ms": 32.318, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 50056, + "latency_ms": 47.214, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 76397, + "latency_ms": 95.2, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 58677, + "latency_ms": 60.548, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 37166, + "latency_ms": 45.388, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 82575, + "latency_ms": 74.166, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 64086, + "latency_ms": 79.285, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 68966, + "latency_ms": 40.081, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37763, + "latency_ms": 41.255, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 91684, + "latency_ms": 62.924, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 56246, + "latency_ms": 37.858, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 60939, + "latency_ms": 44.73, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.4, + "precision_at_5": 0.4, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 1, + "tokens_delivered": 84583, + "latency_ms": 62.542, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 86911, + "latency_ms": 59.301, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 64172, + "latency_ms": 66.343, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 52061, + "latency_ms": 32.694, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 72721, + "latency_ms": 31.966, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 77391, + "latency_ms": 29.975, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53804, + "latency_ms": 24.093, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 67858, + "latency_ms": 32.344, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 155510, + "latency_ms": 44.618, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 51927, + "latency_ms": 38.721, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 44754, + "latency_ms": 26.558, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87223, + "latency_ms": 50.474, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 45358, + "latency_ms": 25.882, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 102374, + "latency_ms": 25.886, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 39464, + "latency_ms": 26.028, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/normalize-arguments.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 40333, + "latency_ms": 24.92, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 45032, + "latency_ms": 17.795, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 64066, + "latency_ms": 45.494, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 84841, + "latency_ms": 46.547, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 70344, + "latency_ms": 46.25, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 65691, + "latency_ms": 46.23, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 84000, + "latency_ms": 45.419, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 45399, + "latency_ms": 31.811, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 73783, + "latency_ms": 31.997, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 46166, + "latency_ms": 40.132, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 80402, + "latency_ms": 47.167, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 39002, + "latency_ms": 19.722, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 75335, + "latency_ms": 26.176, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 46583, + "latency_ms": 26.649, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 60881, + "latency_ms": 32.68, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 41278, + "latency_ms": 31.484, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 62231, + "latency_ms": 33.603, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 63688, + "latency_ms": 26.256, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 46609, + "latency_ms": 25.082, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 61496, + "latency_ms": 34.404, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 35851, + "latency_ms": 24.452, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 96333, + "latency_ms": 37.536, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62246, + "latency_ms": 18.445, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 99388, + "latency_ms": 46.061, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 87406, + "latency_ms": 39.888, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 48035, + "latency_ms": 38.128, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [ + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 4, + "tokens_delivered": 23953, + "latency_ms": 25.935, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 143218, + "latency_ms": 38.729, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 63099, + "latency_ms": 25.82, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 83537, + "latency_ms": 50.056, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 89231, + "latency_ms": 23.853, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 28709, + "latency_ms": 18.112, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 44085, + "latency_ms": 30.718, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53523, + "latency_ms": 25.973, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts", + "test/agent.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 78960, + "latency_ms": 37.31, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 86165, + "latency_ms": 52.586, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 85215, + "latency_ms": 43.408, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 3, + "tokens_delivered": 59030, + "latency_ms": 24.447, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 80040, + "latency_ms": 37.852, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 48574, + "latency_ms": 45.43, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 137075, + "latency_ms": 37.793, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 63528, + "latency_ms": 30.791, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 75580, + "latency_ms": 43.579, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 58669, + "latency_ms": 30.05, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 46865, + "latency_ms": 29.705, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 35083, + "latency_ms": 18.639, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 35641, + "latency_ms": 24.799, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 66786, + "latency_ms": 32.27, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 49650, + "latency_ms": 30.556, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 52150, + "latency_ms": 31.708, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 36292, + "latency_ms": 34.19, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 72027, + "latency_ms": 31.44, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 48302, + "latency_ms": 37.971, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 83669, + "latency_ms": 25.11, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 38479, + "latency_ms": 26.11, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32619, + "latency_ms": 20.471, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 57707, + "latency_ms": 37.597, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 61601, + "latency_ms": 37.021, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 70058, + "latency_ms": 44.134, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/redirects.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 77408, + "latency_ms": 44.876, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 73667, + "latency_ms": 43.971, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 66836, + "latency_ms": 33.58, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 32327, + "latency_ms": 24.815, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 145671, + "latency_ms": 45.246, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 48778, + "latency_ms": 25.834, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 151401, + "latency_ms": 32.402, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 52242, + "latency_ms": 28.814, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28401, + "latency_ms": 20.045, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 87615, + "latency_ms": 32.81, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34148, + "latency_ms": 31.054, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 56450, + "latency_ms": 26.289, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 53717, + "latency_ms": 37.854, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 44547, + "latency_ms": 31.822, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 67602, + "latency_ms": 32.145, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 76447, + "latency_ms": 51.623, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 77055, + "latency_ms": 55.054, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 61652, + "latency_ms": 28.113, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 48300, + "latency_ms": 30.64, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 45872, + "latency_ms": 46.053, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 79869, + "latency_ms": 53.167, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 78365, + "latency_ms": 51.498, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33797, + "latency_ms": 17.128, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 50056, + "latency_ms": 25.178, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 90953, + "latency_ms": 75.628, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 60512, + "latency_ms": 39.038, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 34528, + "latency_ms": 26.938, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 82575, + "latency_ms": 54.783, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 83755, + "latency_ms": 60.216, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 68966, + "latency_ms": 27.608, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37763, + "latency_ms": 26.728, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 90708, + "latency_ms": 39.189, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 56246, + "latency_ms": 17.739, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 63960, + "latency_ms": 24.343, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.2, + "precision_at_10": 0.1, + "recall_at_20": 0.8, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 2, + "tokens_delivered": 85642, + "latency_ms": 45.822, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 86911, + "latency_ms": 39.049, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "test/promise.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 67322, + "latency_ms": 47.35, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/create.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 19586.325, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1301, + "latency_ms": 26377.211, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5208, + "latency_ms": 3018.671, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 1336, + "latency_ms": 8983.701, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5080.98, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 42738.775, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1367, + "latency_ms": 1489.096, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2322, + "latency_ms": 2881.717, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 846, + "latency_ms": 3980.663, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2144.698, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1646, + "latency_ms": 2083.941, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 276, + "latency_ms": 828.604, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 4195.323, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1363, + "latency_ms": 3303.405, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 942.603, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1831, + "latency_ms": 1490.623, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1475.038, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 2778, + "latency_ms": 1087.359, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1410, + "latency_ms": 958.965, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 558, + "latency_ms": 2868.909, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1598, + "latency_ms": 1256.796, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4968, + "latency_ms": 2104.809, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 883.165, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1736.747, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2206, + "latency_ms": 1964.614, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "tsconfig.json" + ], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 1172, + "latency_ms": 1310.632, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1095, + "latency_ms": 925.82, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1248, + "latency_ms": 812.862, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1288, + "latency_ms": 900.754, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1380.196, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1667, + "recall_at_20": 0.25, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 615, + "latency_ms": 847.812, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 2850.873, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1029, + "latency_ms": 1185.811, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1518.534, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1353, + "latency_ms": 986.606, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4286, + "recall_at_20": 1, + "precision_at_20": 0.4286, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 708, + "latency_ms": 1079.382, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 902.765, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 979, + "latency_ms": 1586.618, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 1338, + "latency_ms": 911.26, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1860.782, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1562.901, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 894.106, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/utils/timer.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1558, + "latency_ms": 966.28, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1036.532, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 714, + "latency_ms": 1214.541, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1696.091, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1450.449, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2222, + "recall_at_20": 0.6667, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 1931, + "latency_ms": 791.373, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.25, + "recall_at_20": 0.6667, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 2307, + "latency_ms": 846.319, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 637.988, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4288, + "latency_ms": 1267.182, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 714.987, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 646, + "latency_ms": 1263.494, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 5338, + "latency_ms": 2339.853, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 834, + "latency_ms": 963.19, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1196.804, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2535, + "latency_ms": 842.322, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 14, + "tokens_delivered": 2232, + "latency_ms": 1039.335, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1221, + "latency_ms": 656.246, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "test/redirects.ts" + ], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.3333, + "recall_at_10": 0.3333, + "precision_at_10": 0.3333, + "recall_at_20": 0.3333, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 1675, + "latency_ms": 851.947, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 768.712, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 4715, + "latency_ms": 869.838, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1134, + "latency_ms": 973.227, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1105.689, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4786, + "latency_ms": 925.478, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1340.636, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1016, + "latency_ms": 1390.064, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 739.535, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2679, + "latency_ms": 637.314, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1319, + "latency_ms": 990.127, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1220, + "latency_ms": 942.785, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 819.17, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1457, + "latency_ms": 826.513, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 4214, + "latency_ms": 807.845, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1495, + "latency_ms": 770.764, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.5, + "recall_at_10": 0.6667, + "precision_at_10": 0.5, + "recall_at_20": 0.6667, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 2054, + "latency_ms": 1370.402, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 724, + "latency_ms": 910.268, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1349, + "latency_ms": 1772.115, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 963.557, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 863.409, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 972.717, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 1064, + "latency_ms": 865.554, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 939, + "latency_ms": 1207.831, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 606, + "latency_ms": 1228.424, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 712.012, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 910.009, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 1026.023, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3159, + "latency_ms": 1059.737, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 906.786, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 830.137, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 641.56, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 796.972, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1153, + "latency_ms": 913.049, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 854, + "latency_ms": 1194.085, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1664, + "latency_ms": 641.781, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2594, + "latency_ms": 868.361, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 810.458, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 975, + "latency_ms": 959.417, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "source/core/index.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.125, + "recall_at_20": 0.3333, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 7, + "tokens_delivered": 796, + "latency_ms": 989.699, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1016, + "latency_ms": 946.462, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1037, + "latency_ms": 974.607, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 891.861, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1035, + "latency_ms": 1000.835, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1210, + "latency_ms": 985.752, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 403, + "latency_ms": 826.637, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 842.131, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 833.381, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 834, + "latency_ms": 1298.463, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 15, + "tokens_delivered": 6858, + "latency_ms": 897.224, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1654, + "latency_ms": 1034.32, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 3335, + "latency_ms": 1019.932, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2594, + "latency_ms": 902.572, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [ + "source/core/response.ts" + ], + "missed": [ + "test/encoding.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 5791, + "latency_ms": 985.927, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.75, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 2, + "tokens_delivered": 1675, + "latency_ms": 1227.412, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4439, + "latency_ms": 1055.16, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1655, + "latency_ms": 959.23, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 8203, + "latency_ms": 1157.87, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 2927, + "latency_ms": 1015.086, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts", + "test/abort.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 6443, + "latency_ms": 1254.436, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2733, + "latency_ms": 993.249, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 8, + "tokens_delivered": 6559, + "latency_ms": 796.638, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1665, + "latency_ms": 1014.786, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1693, + "latency_ms": 932.16, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1684, + "latency_ms": 1063.012, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 163, + "latency_ms": 987.584, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 158, + "latency_ms": 957.014, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1659, + "latency_ms": 1025.539, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 489, + "latency_ms": 995.809, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3555, + "latency_ms": 986.263, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3960, + "latency_ms": 780.792, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 2833, + "latency_ms": 1003.281, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3776, + "latency_ms": 998.021, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1672, + "latency_ms": 1044.915, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1663, + "latency_ms": 1253.52, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1661, + "latency_ms": 1042.445, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1675, + "latency_ms": 1045.112, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1660, + "latency_ms": 1056.443, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1657, + "latency_ms": 871.25, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "source/core/response.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 4937, + "latency_ms": 969.415, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 3653, + "latency_ms": 897.137, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2247, + "latency_ms": 865.961, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 1665, + "latency_ms": 1153.386, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1678, + "latency_ms": 878.74, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "source/types.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 1675, + "latency_ms": 1146.454, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts", + "test/timings.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1671, + "latency_ms": 1011.529, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1661, + "latency_ms": 795.272, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 519, + "latency_ms": 782.436, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 5830, + "latency_ms": 1226.76, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1666, + "latency_ms": 907.389, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1713, + "latency_ms": 998.401, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/utils/timer.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 12299, + "latency_ms": 1050.882, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1448, + "latency_ms": 790.423, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3436, + "latency_ms": 821.072, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 892, + "latency_ms": 980.757, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3247, + "latency_ms": 1072.304, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1659, + "latency_ms": 1372.36, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 12, + "tokens_delivered": 1678, + "latency_ms": 1283.176, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4373, + "latency_ms": 993.873, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1677, + "latency_ms": 1124.926, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 6, + "tokens_delivered": 7242, + "latency_ms": 889.189, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1140, + "latency_ms": 1278.848, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1666, + "latency_ms": 881.88, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1660, + "latency_ms": 993.607, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5707, + "latency_ms": 837.084, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1412, + "latency_ms": 781.17, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2113, + "latency_ms": 883.93, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1341, + "latency_ms": 781.526, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1665, + "latency_ms": 838.413, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 153, + "latency_ms": 820.849, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.4, + "recall_at_20": 0.6667, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 2407, + "latency_ms": 816.063, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3863, + "latency_ms": 780.506, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3437, + "latency_ms": 991.965, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 1664, + "latency_ms": 815.843, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3007, + "latency_ms": 1036.43, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2755, + "latency_ms": 978.695, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2834, + "latency_ms": 995.511, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 5412, + "latency_ms": 813.337, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1671, + "latency_ms": 1017.481, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1651, + "latency_ms": 902.052, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1674, + "latency_ms": 1076.316, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "source/core/utils/get-body-size.ts" + ], + "missed": [ + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 5149, + "latency_ms": 1071.836, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1671, + "latency_ms": 912.377, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1312, + "latency_ms": 887.727, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1674, + "latency_ms": 1349.746, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1657, + "latency_ms": 1084.663, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1665, + "latency_ms": 1299.707, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1655, + "latency_ms": 978.628, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2650, + "latency_ms": 931.833, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3589, + "latency_ms": 998.115, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1667, + "latency_ms": 966.798, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1666, + "latency_ms": 968.222, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 1659, + "latency_ms": 958.472, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4701, + "latency_ms": 807.086, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 229, + "latency_ms": 899.74, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1661, + "latency_ms": 973.013, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 1667, + "latency_ms": 997.986, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 4189, + "latency_ms": 898.863, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1677, + "latency_ms": 769.011, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1666, + "latency_ms": 808.429, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1688, + "latency_ms": 1070.771, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 1674, + "latency_ms": 1208.127, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2832, + "latency_ms": 984.801, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1310, + "latency_ms": 798.083, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 1677, + "latency_ms": 1030.318, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0833, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 1663, + "latency_ms": 933.262, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2420, + "latency_ms": 802.584, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 3, + "tokens_delivered": 1675, + "latency_ms": 1008.375, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 220, + "latency_ms": 1002.394, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3593, + "latency_ms": 1014.877, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2724, + "latency_ms": 959.491, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1667, + "latency_ms": 1032.078, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4150, + "latency_ms": 986.38, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/create.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 158, + "latency_ms": 965.988, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "source/types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "missed": [ + "test/create.ts", + "test/extend.types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.6, + "recall_at_4000_tokens": 0.6, + "recall_at_16000_tokens": 0.6, + "first_hit_rank": 8, + "tokens_delivered": 1655, + "latency_ms": 1073.904, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1245, + "latency_ms": 1022.442, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 1680, + "latency_ms": 863.954, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "source/core/index.ts", + "test/create.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 58679, + "latency_ms": 31.298, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 89852, + "latency_ms": 24.662, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 82176, + "latency_ms": 24.776, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 57240, + "latency_ms": 26.661, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73871, + "latency_ms": 25.224, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/gzip.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 162539, + "latency_ms": 23.804, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 87852, + "latency_ms": 23.599, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 73650, + "latency_ms": 26.081, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 103569, + "latency_ms": 24.676, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 71989, + "latency_ms": 24.959, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 162719, + "latency_ms": 25.282, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55161, + "latency_ms": 26.08, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 49756, + "latency_ms": 26.704, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 69499, + "latency_ms": 25.388, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 57290, + "latency_ms": 26.249, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 93628, + "latency_ms": 24.75, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 57286, + "latency_ms": 25.809, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 71767, + "latency_ms": 25.276, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 91285, + "latency_ms": 25.741, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 77785, + "latency_ms": 27.238, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 88524, + "latency_ms": 25.167, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 53380, + "latency_ms": 26.004, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 91140, + "latency_ms": 27.08, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 76834, + "latency_ms": 25.305, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 101370, + "latency_ms": 25.251, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "package.json" + ], + "missed": [ + "tsconfig.json" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 76435, + "latency_ms": 27.45, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 76435, + "latency_ms": 26.671, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 87924, + "latency_ms": 28.176, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 76166, + "latency_ms": 29.259, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 68859, + "latency_ms": 31.331, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 71393, + "latency_ms": 29.467, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 70103, + "latency_ms": 30.768, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 51379, + "latency_ms": 31.236, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 103622, + "latency_ms": 29.437, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70281, + "latency_ms": 30.769, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [ + "source/types.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 103976, + "latency_ms": 28.742, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 87647, + "latency_ms": 29.189, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54612, + "latency_ms": 30.439, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.75, + "precision_at_10": 0.3, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 50065, + "latency_ms": 30.98, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 163314, + "latency_ms": 30.245, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 72182, + "latency_ms": 29.494, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 89284, + "latency_ms": 31.258, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 103622, + "latency_ms": 27.52, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 57236, + "latency_ms": 31.23, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts" + ], + "missed": [ + "source/as-promise/types.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 56354, + "latency_ms": 31.826, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 77422, + "latency_ms": 30.519, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 84288, + "latency_ms": 29.29, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 89510, + "latency_ms": 27.631, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 90189, + "latency_ms": 27.988, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/arguments.ts", + "test/error.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 57196, + "latency_ms": 30.289, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [ + "source/core/options.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 82777, + "latency_ms": 31.115, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/core.ts", + "source/core/index.ts" + ], + "missed": [ + "test/http.ts", + "test/promise.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 55092, + "latency_ms": 33.448, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 162744, + "latency_ms": 30.257, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 55701, + "latency_ms": 32.067, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "test/retry.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/timed-out.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 82287, + "latency_ms": 30.086, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [ + "test/stream.ts" + ], + "missed": [ + "source/core/utils/get-body-size.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 69971, + "latency_ms": 30.907, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 49545, + "latency_ms": 31.773, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 54988, + "latency_ms": 30.626, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 51355, + "latency_ms": 31.703, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 69534, + "latency_ms": 30.362, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 55421, + "latency_ms": 35.581, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 55679, + "latency_ms": 30.934, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 51051, + "latency_ms": 29.46, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82219, + "latency_ms": 30.297, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "test/error.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 54539, + "latency_ms": 30.658, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 94625, + "latency_ms": 28.343, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [], + "missed": [ + "test/progress.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73537, + "latency_ms": 29.724, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 71935, + "latency_ms": 29.597, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/https.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 58746, + "latency_ms": 28.863, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82168, + "latency_ms": 29.897, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 57062, + "latency_ms": 28.688, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 87789, + "latency_ms": 28.306, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [ + "test/post.ts" + ], + "missed": [ + "source/core/utils/get-body-size.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 88149, + "latency_ms": 30.444, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 54889, + "latency_ms": 30.588, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 56081, + "latency_ms": 29.87, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [ + "test/early-307-redirects.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 163538, + "latency_ms": 29.118, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 77420, + "latency_ms": 30.878, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 161120, + "latency_ms": 30.647, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 77692, + "latency_ms": 34.381, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 73848, + "latency_ms": 28.506, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 95064, + "latency_ms": 29.464, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 82287, + "latency_ms": 28.456, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [], + "missed": [ + "test/cookies.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 77134, + "latency_ms": 29.684, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 79062, + "latency_ms": 28.426, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 69515, + "latency_ms": 28.586, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 72804, + "latency_ms": 34.305, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82400, + "latency_ms": 28.591, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 81765, + "latency_ms": 28.568, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts" + ], + "missed": [ + "test/headers.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 72116, + "latency_ms": 28.747, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 53420, + "latency_ms": 27.834, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 55889, + "latency_ms": 27.509, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 82176, + "latency_ms": 27.001, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts", + "test/stream.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82513, + "latency_ms": 27.992, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 82131, + "latency_ms": 26.806, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [ + "source/create.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 57497, + "latency_ms": 37.31, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 90031, + "latency_ms": 28.26, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 72711, + "latency_ms": 28.128, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 56960, + "latency_ms": 29.873, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 93102, + "latency_ms": 27.814, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 86056, + "latency_ms": 26.452, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 74001, + "latency_ms": 27.032, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 73071, + "latency_ms": 26.61, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 97394, + "latency_ms": 25.976, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts", + "test/cache.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82168, + "latency_ms": 25.901, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [ + "source/create.ts", + "test/arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 82112, + "latency_ms": 26.479, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "test/create.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "missed": [ + "source/types.ts", + "test/extend.types.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.6, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 103627, + "latency_ms": 25.254, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 87367, + "latency_ms": 28.497, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 70574, + "latency_ms": 28.818, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-043c9501b851", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/create.ts" + ], + "found": [ + "test/create.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 17695, + "latency_ms": 14.455, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-05cb01405a70", + "path_leak": false, + "required_files": [ + "test/arguments.ts" + ], + "found": [], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29330, + "latency_ms": 12.857, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-06029e0b758e", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44537, + "latency_ms": 12.397, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-06a2d3d7d8d4", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 30339, + "latency_ms": 12.909, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-0703318d3bb6", + "path_leak": true, + "required_files": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts", + "test/encoding.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28268, + "latency_ms": 12.567, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-071ea0749d81", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/timed-out.ts", + "test/gzip.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts", + "test/gzip.ts" + ], + "missed": [ + "source/core/timed-out.ts", + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 37092, + "latency_ms": 12.224, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-0863bcd5e90e", + "path_leak": true, + "required_files": [ + "test/timeout.ts" + ], + "found": [], + "missed": [ + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24761, + "latency_ms": 13.948, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-08d7a0f3285d", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [ + "test/arguments.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 32998, + "latency_ms": 13.936, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-0a16a9ca7de3", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 45500, + "latency_ms": 12.758, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1018c2029eea", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "found": [ + "source/core/options.ts", + "test/normalize-arguments.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 40700, + "latency_ms": 12.591, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-11a2202de328", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/abort.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 63090, + "latency_ms": 12.775, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-15395ee87e6d", + "path_leak": true, + "required_files": [ + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29834, + "latency_ms": 12.34, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-157e02bcb613", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/normalize-arguments.ts", + "test/pagination.ts" + ], + "found": [ + "test/normalize-arguments.ts" + ], + "missed": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 19371, + "latency_ms": 12.314, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1b208df66716", + "path_leak": true, + "required_files": [ + "test/stream.ts" + ], + "found": [], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26240, + "latency_ms": 12.681, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1e1e50647e93", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 41881, + "latency_ms": 14.658, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-1e49781eff61", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 40853, + "latency_ms": 14.073, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-226cc3995f6e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 41220, + "latency_ms": 14.896, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2453e5e4213f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/http.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/http.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 19500, + "latency_ms": 12.757, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2527bf690fd5", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 40496, + "latency_ms": 12.785, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-26fffe94618b", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18203, + "latency_ms": 14.766, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2ab94fd4daf6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24438, + "latency_ms": 15.142, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-2ccc4c2e8bd5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20036, + "latency_ms": 13.861, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-30b3b795580d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20481, + "latency_ms": 14.516, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-32070619a547", + "path_leak": true, + "required_files": [ + "test/https.ts" + ], + "found": [ + "test/https.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 27897, + "latency_ms": 16.747, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-398c11ae502c", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [ + "test/timings.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 48003, + "latency_ms": 17.012, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3b3ea6776b4f", + "path_leak": false, + "required_files": [ + "package.json", + "tsconfig.json" + ], + "found": [ + "tsconfig.json" + ], + "missed": [ + "package.json" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 44800, + "latency_ms": 14.831, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3cc40b549c4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25732, + "latency_ms": 18.249, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3d66aecacd6d", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19504, + "latency_ms": 15.095, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-3e9d3af606c3", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29947, + "latency_ms": 13.52, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-408e22aed993", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17080, + "latency_ms": 13.343, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-439fb82d2a07", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "source/core/response.ts", + "test/pagination.ts" + ], + "found": [ + "source/core/index.ts", + "source/core/options.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 42253, + "latency_ms": 13.116, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-462bc6300150", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 50293, + "latency_ms": 12.41, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-47c1afedc77b", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 17, + "tokens_delivered": 16939, + "latency_ms": 12.647, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-47c3155e4bdf", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/http.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/http.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 64594, + "latency_ms": 12.498, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-4cce4de19ed5", + "path_leak": false, + "required_files": [ + "source/as-promise/types.ts" + ], + "found": [ + "source/as-promise/types.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 12, + "tokens_delivered": 20438, + "latency_ms": 12.928, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-4d5168cc7c58", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/types.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19771, + "latency_ms": 12.186, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-4e7567965dbc", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timings.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 32900, + "latency_ms": 13.315, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-5028c11abd21", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 36724, + "latency_ms": 13.943, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-50ef99a9856c", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "missed": [ + "test/agent.ts", + "test/retry.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 29961, + "latency_ms": 13.689, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-5772bf24dc7d", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/hooks.ts", + "test/normalize-arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29680, + "latency_ms": 13.529, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-59ae3b062f53", + "path_leak": true, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 45651, + "latency_ms": 13.586, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-60a441939af3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 20651, + "latency_ms": 13.811, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-61d5e3b0ebc5", + "path_leak": false, + "required_files": [ + "source/core/utils/timer.ts", + "test/timings.ts" + ], + "found": [ + "source/core/utils/timer.ts" + ], + "missed": [ + "test/timings.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 26908, + "latency_ms": 13.636, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-61d6f610ffa6", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/agent.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14053, + "latency_ms": 15.432, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-676be6d85481", + "path_leak": true, + "required_files": [ + "source/as-promise/core.ts", + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "found": [ + "source/as-promise/core.ts" + ], + "missed": [ + "source/as-promise/types.ts", + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 23093, + "latency_ms": 15.2, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-67d5039ff5a5", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 34438, + "latency_ms": 13.371, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6961284172c4", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/agent.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "test/agent.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 40983, + "latency_ms": 14.075, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6a544a392baf", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 7, + "tokens_delivered": 38400, + "latency_ms": 13.534, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6ae3e7fc364b", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "source/core/index.ts", + "test/retry.ts" + ], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 56327, + "latency_ms": 12.056, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-6e1d9f97465d", + "path_leak": true, + "required_files": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/error.ts", + "test/helpers.ts", + "test/helpers/invalid-url.ts" + ], + "found": [ + "test/error.ts", + "test/helpers.ts" + ], + "missed": [ + "source/core/errors.ts", + "test/arguments.ts", + "test/helpers/invalid-url.ts" + ], + "recall_at_5": 0.2, + "precision_at_5": 0.2, + "recall_at_10": 0.4, + "precision_at_10": 0.2, + "recall_at_20": 0.4, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.2, + "recall_at_16000_tokens": 0.4, + "first_hit_rank": 2, + "tokens_delivered": 27556, + "latency_ms": 12.126, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-724d5923816b", + "path_leak": false, + "required_files": [ + "source/core/options.ts", + "test/https.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/https.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31766, + "latency_ms": 12.044, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-728aef989c1a", + "path_leak": false, + "required_files": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/http.ts", + "test/promise.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [ + "source/as-promise/core.ts", + "source/core/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 9, + "tokens_delivered": 25946, + "latency_ms": 11.578, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-74e3167b705b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [ + "test/abort.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 61131, + "latency_ms": 12.263, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-78b105b5ebcf", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43051, + "latency_ms": 12.224, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7a920647efa2", + "path_leak": true, + "required_files": [ + "source/core/timed-out.ts", + "test/retry.ts", + "test/timeout.ts" + ], + "found": [ + "source/core/timed-out.ts", + "test/timeout.ts" + ], + "missed": [ + "test/retry.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 55773, + "latency_ms": 12.614, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7acd3801ec7d", + "path_leak": true, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43907, + "latency_ms": 13.736, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7b19e8f32c90", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 26684, + "latency_ms": 14.022, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-7e1de4240dad", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [ + "test/timeout.ts" + ], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 28696, + "latency_ms": 13.018, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-88622701c704", + "path_leak": false, + "required_files": [ + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts" + ], + "missed": [ + "test/pagination.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 29633, + "latency_ms": 13.197, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-88b32eade80f", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts", + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 31568, + "latency_ms": 11.726, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-8d3412abb8aa", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/arguments.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 32483, + "latency_ms": 12.32, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-8f775c71c41a", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/create.ts", + "test/pagination.ts" + ], + "found": [ + "source/create.ts", + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 5, + "tokens_delivered": 16998, + "latency_ms": 12.811, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-91aa0ac9e676", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 28802, + "latency_ms": 12.531, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-92b378e5fd95", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/abort.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/abort.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18220, + "latency_ms": 12.369, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-934211f951bf", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [ + "source/as-promise/index.ts" + ], + "missed": [ + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 34057, + "latency_ms": 13.151, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9725fbd109a6", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27444, + "latency_ms": 12.257, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-98ca6181a039", + "path_leak": true, + "required_files": [ + "test/progress.ts" + ], + "found": [ + "test/progress.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 43111, + "latency_ms": 12.388, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-99e8d134bd9a", + "path_leak": true, + "required_files": [ + "test/http.ts" + ], + "found": [ + "test/http.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 36388, + "latency_ms": 12.543, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9a309bdbe7e2", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/https.ts" + ], + "found": [ + "source/core/index.ts", + "test/https.ts" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 26052, + "latency_ms": 11.647, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9ab4cf991fd5", + "path_leak": false, + "required_files": [ + "source/core/response.ts" + ], + "found": [], + "missed": [ + "source/core/response.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34828, + "latency_ms": 12.268, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9e15d887da3b", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 47799, + "latency_ms": 12.117, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-9ec6ff02a189", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15060, + "latency_ms": 13.624, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-a2812de3baa0", + "path_leak": false, + "required_files": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "found": [], + "missed": [ + "source/core/utils/get-body-size.ts", + "test/post.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35728, + "latency_ms": 12.873, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-a59fac415ac0", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "source/core/options.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30113, + "latency_ms": 12.224, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-ac5f67dd4b92", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/cache.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 26343, + "latency_ms": 12.512, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-aee9249444c4", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/early-307-redirects.ts", + "test/redirects.ts" + ], + "found": [ + "source/core/index.ts", + "test/redirects.ts" + ], + "missed": [ + "test/early-307-redirects.ts" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 66990, + "latency_ms": 12.34, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-af928f6d3974", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 33984, + "latency_ms": 12.096, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-b17012597e4d", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 70809, + "latency_ms": 12.633, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-b671480715db", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16148, + "latency_ms": 12.241, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-bf39d2c1c58f", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22493, + "latency_ms": 13.136, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-bf84d3697537", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 43364, + "latency_ms": 12.03, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c079b9366ff2", + "path_leak": false, + "required_files": [ + "source/core/index.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25107, + "latency_ms": 12.909, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c09b9857d1b4", + "path_leak": true, + "required_files": [ + "test/cookies.ts" + ], + "found": [ + "test/cookies.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 34784, + "latency_ms": 12.711, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c81a61159473", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/error.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21815, + "latency_ms": 12.913, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-c97ce7cbde86", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 39157, + "latency_ms": 12.377, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-cbc890292f40", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21610, + "latency_ms": 12.445, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-cc434bc52111", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/no-body-status-codes.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/no-body-status-codes.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 8, + "tokens_delivered": 36075, + "latency_ms": 12.181, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-d1d4ed2f6242", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24832, + "latency_ms": 12.691, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-d65d0caf627e", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "source/core/options.ts", + "test/headers.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "source/core/options.ts", + "test/headers.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 32152, + "latency_ms": 12.337, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-d8c00cfa388b", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 25999, + "latency_ms": 12.156, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-da4769e98880", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16382, + "latency_ms": 13.531, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-dad6a91b3d11", + "path_leak": true, + "required_files": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts", + "test/infinite-loop-issue.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37626, + "latency_ms": 12.086, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-dc4f1e323dbd", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/stream.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/stream.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 57177, + "latency_ms": 11.545, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-dfc54d9e57b7", + "path_leak": false, + "required_files": [ + "package.json" + ], + "found": [ + "package.json" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 34640, + "latency_ms": 12.504, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e02845f1aa73", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24940, + "latency_ms": 12.298, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e09a9bd1ca97", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "source/core/index.ts" + ], + "missed": [ + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 39937, + "latency_ms": 13.503, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e1099fbc2aa9", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [ + "source/core/options.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 42625, + "latency_ms": 11.173, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e4aeb8e3bfc4", + "path_leak": true, + "required_files": [ + "test/pagination.ts" + ], + "found": [ + "test/pagination.ts" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 30034, + "latency_ms": 12.254, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e5659d401a4a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "source/core/utils/is-unix-socket-url.ts", + "test/get-unix-socket-path.ts" + ], + "found": [ + "source/core/utils/is-unix-socket-url.ts" + ], + "missed": [ + "source/core/index.ts", + "test/get-unix-socket-path.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 26779, + "latency_ms": 12.103, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-e899c0736f6a", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/retry.ts" + ], + "found": [ + "test/retry.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 40775, + "latency_ms": 11.964, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-eb045bfc324d", + "path_leak": false, + "required_files": [ + "source/core/options.ts" + ], + "found": [], + "missed": [ + "source/core/options.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13037, + "latency_ms": 12.228, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-ecb05343dea3", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23583, + "latency_ms": 12.394, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f0045640d12a", + "path_leak": false, + "required_files": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/hooks.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27860, + "latency_ms": 13.644, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f2f8cb29fe9f", + "path_leak": true, + "required_files": [ + "source/core/index.ts", + "test/cache.ts" + ], + "found": [], + "missed": [ + "source/core/index.ts", + "test/cache.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24606, + "latency_ms": 12.406, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f44ef4388e0b", + "path_leak": true, + "required_files": [ + "source/create.ts", + "test/arguments.ts" + ], + "found": [], + "missed": [ + "source/create.ts", + "test/arguments.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28939, + "latency_ms": 11.978, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-f7ab6e9ffda6", + "path_leak": true, + "required_files": [ + "source/types.ts", + "test/create.ts", + "test/extend.types.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "found": [ + "test/extend.types.ts" + ], + "missed": [ + "source/types.ts", + "test/create.ts", + "test/hooks.ts", + "test/timeout.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.2, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.2, + "first_hit_rank": 11, + "tokens_delivered": 24896, + "latency_ms": 12.539, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-fb8641856a13", + "path_leak": false, + "required_files": [ + "source/core/index.ts", + "test/hooks.ts" + ], + "found": [ + "test/hooks.ts" + ], + "missed": [ + "source/core/index.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 56437, + "latency_ms": 12.634, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "got-ff918fb6dedb", + "path_leak": true, + "required_files": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "found": [], + "missed": [ + "source/as-promise/index.ts", + "test/promise.ts" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17646, + "latency_ms": 14.388, + "indexed_revision_matches": true + } + ], + "tier": "small", + "tier_code_files": 85, + "tier_provenance": "Stamped after the fact from a measured file count; the run predates tier stamping. Tier is a property of the repository, not of the run." +} diff --git a/benchmarks/context-retrieval/results/small-gin-13arm.json b/benchmarks/context-retrieval/results/small-gin-13arm.json new file mode 100644 index 00000000..66eced90 --- /dev/null +++ b/benchmarks/context-retrieval/results/small-gin-13arm.json @@ -0,0 +1,25841 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "corpus_counts": { + "cases": 76, + "path_leak": 23, + "baseline_blind": 53, + "multi_file": 54 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "providers": [ + { + "provider_id": "agent-default", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.6042, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.7566, + "full_recall_at_10": 0.6842, + "mean_recall_at_20": 0.7917, + "full_recall_at_20": 0.7368, + "mean_recall_at_1000_tokens": 0.0066, + "mean_recall_at_4000_tokens": 0.1557, + "mean_recall_at_16000_tokens": 0.5143, + "mean_precision_at_10": 0.1632, + "zero_hit_rate": 0.1579, + "median_tokens_delivered": 67885.5, + "median_latency_ms": 54.376, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.1563, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2292, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3403, + "full_recall_at_20": 0.1667, + "mean_recall_at_1000_tokens": 0.0208, + "mean_recall_at_4000_tokens": 0.0833, + "mean_recall_at_16000_tokens": 0.1563, + "mean_precision_at_10": 0.1, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 67258.5, + "median_latency_ms": 47.445, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.8109, + "full_recall_at_5": 0.7308, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1891, + "mean_recall_at_16000_tokens": 0.6795, + "mean_precision_at_10": 0.1923, + "zero_hit_rate": 0, + "median_tokens_delivered": 69842, + "median_latency_ms": 55.313, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.7826, + "full_recall_at_5": 0.6522, + "mean_recall_at_10": 0.9058, + "full_recall_at_10": 0.8261, + "mean_recall_at_20": 0.9275, + "full_recall_at_20": 0.8696, + "mean_recall_at_1000_tokens": 0.0217, + "mean_recall_at_4000_tokens": 0.3768, + "mean_recall_at_16000_tokens": 0.8261, + "mean_precision_at_10": 0.1913, + "zero_hit_rate": 0, + "median_tokens_delivered": 63498, + "median_latency_ms": 54.792, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.5267, + "full_recall_at_5": 0.434, + "mean_recall_at_10": 0.6918, + "full_recall_at_10": 0.6226, + "mean_recall_at_20": 0.7327, + "full_recall_at_20": 0.6792, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0597, + "mean_recall_at_16000_tokens": 0.3789, + "mean_precision_at_10": 0.1509, + "zero_hit_rate": 0.2264, + "median_tokens_delivered": 71198, + "median_latency_ms": 53.71, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 71532.3, + "peak_rss_mb": 71710.4, + "delta_rss_mb": 178.2, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "ripgrep", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.591, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.7325, + "full_recall_at_10": 0.6447, + "mean_recall_at_20": 0.7917, + "full_recall_at_20": 0.7237, + "mean_recall_at_1000_tokens": 0.0066, + "mean_recall_at_4000_tokens": 0.1118, + "mean_recall_at_16000_tokens": 0.5296, + "mean_precision_at_10": 0.1579, + "zero_hit_rate": 0.1579, + "median_tokens_delivered": 70681, + "median_latency_ms": 95.9285, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.1493, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2431, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3542, + "full_recall_at_20": 0.1667, + "mean_recall_at_1000_tokens": 0.0208, + "mean_recall_at_4000_tokens": 0.0417, + "mean_recall_at_16000_tokens": 0.1493, + "mean_precision_at_10": 0.1042, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 67270.5, + "median_latency_ms": 85.7185, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.7949, + "full_recall_at_5": 0.7308, + "mean_recall_at_10": 0.9583, + "full_recall_at_10": 0.9423, + "mean_recall_at_20": 0.9936, + "full_recall_at_20": 0.9808, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1442, + "mean_recall_at_16000_tokens": 0.7051, + "mean_precision_at_10": 0.1827, + "zero_hit_rate": 0, + "median_tokens_delivered": 71145, + "median_latency_ms": 96.915, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.6522, + "full_recall_at_5": 0.4783, + "mean_recall_at_10": 0.8261, + "full_recall_at_10": 0.6957, + "mean_recall_at_20": 0.9275, + "full_recall_at_20": 0.8261, + "mean_recall_at_1000_tokens": 0.0217, + "mean_recall_at_4000_tokens": 0.1522, + "mean_recall_at_16000_tokens": 0.6377, + "mean_precision_at_10": 0.1739, + "zero_hit_rate": 0, + "median_tokens_delivered": 69644, + "median_latency_ms": 96.282, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.5645, + "full_recall_at_5": 0.5094, + "mean_recall_at_10": 0.6918, + "full_recall_at_10": 0.6226, + "mean_recall_at_20": 0.7327, + "full_recall_at_20": 0.6792, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0943, + "mean_recall_at_16000_tokens": 0.4827, + "mean_precision_at_10": 0.1509, + "zero_hit_rate": 0.2264, + "median_tokens_delivered": 71198, + "median_latency_ms": 90.607, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 71610.6, + "peak_rss_mb": 71850.3, + "delta_rss_mb": 239.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "keyword-search", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.591, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.7325, + "full_recall_at_10": 0.6447, + "mean_recall_at_20": 0.7917, + "full_recall_at_20": 0.7237, + "mean_recall_at_1000_tokens": 0.0066, + "mean_recall_at_4000_tokens": 0.1118, + "mean_recall_at_16000_tokens": 0.5296, + "mean_precision_at_10": 0.1579, + "zero_hit_rate": 0.1579, + "median_tokens_delivered": 70681, + "median_latency_ms": 36.6225, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.1493, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2431, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.3542, + "full_recall_at_20": 0.1667, + "mean_recall_at_1000_tokens": 0.0208, + "mean_recall_at_4000_tokens": 0.0417, + "mean_recall_at_16000_tokens": 0.1493, + "mean_precision_at_10": 0.1042, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 67270.5, + "median_latency_ms": 33.843, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.7949, + "full_recall_at_5": 0.7308, + "mean_recall_at_10": 0.9583, + "full_recall_at_10": 0.9423, + "mean_recall_at_20": 0.9936, + "full_recall_at_20": 0.9808, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1442, + "mean_recall_at_16000_tokens": 0.7051, + "mean_precision_at_10": 0.1827, + "zero_hit_rate": 0, + "median_tokens_delivered": 71145, + "median_latency_ms": 38.7935, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.6522, + "full_recall_at_5": 0.4783, + "mean_recall_at_10": 0.8261, + "full_recall_at_10": 0.6957, + "mean_recall_at_20": 0.9275, + "full_recall_at_20": 0.8261, + "mean_recall_at_1000_tokens": 0.0217, + "mean_recall_at_4000_tokens": 0.1522, + "mean_recall_at_16000_tokens": 0.6377, + "mean_precision_at_10": 0.1739, + "zero_hit_rate": 0, + "median_tokens_delivered": 69644, + "median_latency_ms": 35.36, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.5645, + "full_recall_at_5": 0.5094, + "mean_recall_at_10": 0.6918, + "full_recall_at_10": 0.6226, + "mean_recall_at_20": 0.7327, + "full_recall_at_20": 0.6792, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0943, + "mean_recall_at_16000_tokens": 0.4827, + "mean_precision_at_10": 0.1509, + "zero_hit_rate": 0.2264, + "median_tokens_delivered": 71198, + "median_latency_ms": 37.3, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 70835.5, + "peak_rss_mb": 71149, + "delta_rss_mb": 313.4, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "filename-match", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.2138, + "full_recall_at_5": 0.1711, + "mean_recall_at_10": 0.2467, + "full_recall_at_10": 0.2105, + "mean_recall_at_20": 0.2664, + "full_recall_at_20": 0.2368, + "mean_recall_at_1000_tokens": 0.0318, + "mean_recall_at_4000_tokens": 0.0844, + "mean_recall_at_16000_tokens": 0.1612, + "mean_precision_at_10": 0.0919, + "zero_hit_rate": 0.6974, + "median_tokens_delivered": 217.5, + "median_latency_ms": 10.548, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.066, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.066, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.066, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.0104, + "mean_recall_at_4000_tokens": 0.0313, + "mean_recall_at_16000_tokens": 0.066, + "mean_precision_at_10": 0.0708, + "zero_hit_rate": 0.8333, + "median_tokens_delivered": 0, + "median_latency_ms": 5.976, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.2821, + "full_recall_at_5": 0.25, + "mean_recall_at_10": 0.3301, + "full_recall_at_10": 0.3077, + "mean_recall_at_20": 0.359, + "full_recall_at_20": 0.3462, + "mean_recall_at_1000_tokens": 0.0417, + "mean_recall_at_4000_tokens": 0.109, + "mean_recall_at_16000_tokens": 0.2051, + "mean_precision_at_10": 0.1016, + "zero_hit_rate": 0.6346, + "median_tokens_delivered": 500.5, + "median_latency_ms": 10.8235, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.7065, + "full_recall_at_5": 0.5652, + "mean_recall_at_10": 0.8152, + "full_recall_at_10": 0.6957, + "mean_recall_at_20": 0.8804, + "full_recall_at_20": 0.7826, + "mean_recall_at_1000_tokens": 0.1051, + "mean_recall_at_4000_tokens": 0.279, + "mean_recall_at_16000_tokens": 0.5326, + "mean_precision_at_10": 0.3037, + "zero_hit_rate": 0, + "median_tokens_delivered": 25055, + "median_latency_ms": 11.353, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0, + "full_recall_at_5": 0, + "mean_recall_at_10": 0, + "full_recall_at_10": 0, + "mean_recall_at_20": 0, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0, + "zero_hit_rate": 1, + "median_tokens_delivered": 0, + "median_latency_ms": 6.403, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 69759.1, + "peak_rss_mb": 69760.6, + "delta_rss_mb": 1.5, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "zoekt", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.4616, + "full_recall_at_5": 0.3289, + "mean_recall_at_10": 0.5932, + "full_recall_at_10": 0.4605, + "mean_recall_at_20": 0.7149, + "full_recall_at_20": 0.6184, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0822, + "mean_recall_at_16000_tokens": 0.4518, + "mean_precision_at_10": 0.1215, + "zero_hit_rate": 0.1842, + "median_tokens_delivered": 62006, + "median_latency_ms": 276.58, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.1076, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.2535, + "full_recall_at_10": 0.0833, + "mean_recall_at_20": 0.3472, + "full_recall_at_20": 0.1667, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0521, + "mean_recall_at_16000_tokens": 0.1597, + "mean_precision_at_10": 0.075, + "zero_hit_rate": 0.4583, + "median_tokens_delivered": 63165.5, + "median_latency_ms": 245.627, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.625, + "full_recall_at_5": 0.4808, + "mean_recall_at_10": 0.75, + "full_recall_at_10": 0.6346, + "mean_recall_at_20": 0.8846, + "full_recall_at_20": 0.8269, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0962, + "mean_recall_at_16000_tokens": 0.5865, + "mean_precision_at_10": 0.1429, + "zero_hit_rate": 0.0577, + "median_tokens_delivered": 62006, + "median_latency_ms": 283.295, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.4928, + "full_recall_at_5": 0.4348, + "mean_recall_at_10": 0.5942, + "full_recall_at_10": 0.4348, + "mean_recall_at_20": 0.7391, + "full_recall_at_20": 0.6087, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0217, + "mean_recall_at_16000_tokens": 0.5906, + "mean_precision_at_10": 0.113, + "zero_hit_rate": 0.1304, + "median_tokens_delivered": 53700, + "median_latency_ms": 277.613, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.4481, + "full_recall_at_5": 0.283, + "mean_recall_at_10": 0.5928, + "full_recall_at_10": 0.4717, + "mean_recall_at_20": 0.7044, + "full_recall_at_20": 0.6226, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.1085, + "mean_recall_at_16000_tokens": 0.3915, + "mean_precision_at_10": 0.1252, + "zero_hit_rate": 0.2075, + "median_tokens_delivered": 67521, + "median_latency_ms": 275.547, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 69654.8, + "peak_rss_mb": 71611.4, + "delta_rss_mb": 1956.6, + "scope": "system-wide-upper-bound", + "index_bytes_added": 186880000 + } + }, + { + "provider_id": "ast-grep", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.2127, + "full_recall_at_5": 0.0789, + "mean_recall_at_10": 0.2917, + "full_recall_at_10": 0.1184, + "mean_recall_at_20": 0.364, + "full_recall_at_20": 0.1842, + "mean_recall_at_1000_tokens": 0.0164, + "mean_recall_at_4000_tokens": 0.0713, + "mean_recall_at_16000_tokens": 0.1721, + "mean_precision_at_10": 0.0956, + "zero_hit_rate": 0.4474, + "median_tokens_delivered": 28907.5, + "median_latency_ms": 233.0605, + "payload_kind": [ + "whole-files" + ], + "unavailable": 12 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.0972, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.1042, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1736, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0.0104, + "mean_recall_at_4000_tokens": 0.0174, + "mean_recall_at_16000_tokens": 0.0938, + "mean_precision_at_10": 0.0833, + "zero_hit_rate": 0.625, + "median_tokens_delivered": 15165, + "median_latency_ms": 203.3775, + "payload_kind": [ + "whole-files" + ], + "unavailable": 7 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.266, + "full_recall_at_5": 0.1154, + "mean_recall_at_10": 0.3782, + "full_recall_at_10": 0.1731, + "mean_recall_at_20": 0.4519, + "full_recall_at_20": 0.2692, + "mean_recall_at_1000_tokens": 0.0192, + "mean_recall_at_4000_tokens": 0.0962, + "mean_recall_at_16000_tokens": 0.2083, + "mean_precision_at_10": 0.1013, + "zero_hit_rate": 0.3654, + "median_tokens_delivered": 33408.5, + "median_latency_ms": 250.8155, + "payload_kind": [ + "whole-files" + ], + "unavailable": 5 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.2138, + "full_recall_at_5": 0.087, + "mean_recall_at_10": 0.3225, + "full_recall_at_10": 0.1739, + "mean_recall_at_20": 0.4239, + "full_recall_at_20": 0.2609, + "mean_recall_at_1000_tokens": 0.0543, + "mean_recall_at_4000_tokens": 0.0543, + "mean_recall_at_16000_tokens": 0.1703, + "mean_precision_at_10": 0.1338, + "zero_hit_rate": 0.3913, + "median_tokens_delivered": 33240, + "median_latency_ms": 228.73, + "payload_kind": [ + "whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.2123, + "full_recall_at_5": 0.0755, + "mean_recall_at_10": 0.2783, + "full_recall_at_10": 0.0943, + "mean_recall_at_20": 0.3381, + "full_recall_at_20": 0.1509, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0786, + "mean_recall_at_16000_tokens": 0.173, + "mean_precision_at_10": 0.079, + "zero_hit_rate": 0.4717, + "median_tokens_delivered": 27416, + "median_latency_ms": 236.484, + "payload_kind": [ + "whole-files" + ], + "unavailable": 12 + } + }, + "resources": { + "baseline_rss_mb": 71520, + "peak_rss_mb": 71952.7, + "delta_rss_mb": 432.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "semble", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.4616, + "full_recall_at_5": 0.1974, + "mean_recall_at_10": 0.6206, + "full_recall_at_10": 0.3947, + "mean_recall_at_20": 0.7577, + "full_recall_at_20": 0.5921, + "mean_recall_at_1000_tokens": 0.4397, + "mean_recall_at_4000_tokens": 0.7511, + "mean_recall_at_16000_tokens": 0.7577, + "mean_precision_at_10": 0.1118, + "zero_hit_rate": 0.0921, + "median_tokens_delivered": 3848, + "median_latency_ms": 702.0515, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.2118, + "full_recall_at_5": 0.0417, + "mean_recall_at_10": 0.3333, + "full_recall_at_10": 0.1667, + "mean_recall_at_20": 0.5174, + "full_recall_at_20": 0.2917, + "mean_recall_at_1000_tokens": 0.184, + "mean_recall_at_4000_tokens": 0.5174, + "mean_recall_at_16000_tokens": 0.5174, + "mean_precision_at_10": 0.0792, + "zero_hit_rate": 0.2917, + "median_tokens_delivered": 3868, + "median_latency_ms": 742.219, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.5769, + "full_recall_at_5": 0.2692, + "mean_recall_at_10": 0.7532, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.8686, + "full_recall_at_20": 0.7308, + "mean_recall_at_1000_tokens": 0.5577, + "mean_recall_at_4000_tokens": 0.859, + "mean_recall_at_16000_tokens": 0.8686, + "mean_precision_at_10": 0.1269, + "zero_hit_rate": 0, + "median_tokens_delivered": 3848, + "median_latency_ms": 697.937, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.6594, + "full_recall_at_5": 0.3913, + "mean_recall_at_10": 0.7681, + "full_recall_at_10": 0.5217, + "mean_recall_at_20": 0.8116, + "full_recall_at_20": 0.6087, + "mean_recall_at_1000_tokens": 0.6087, + "mean_recall_at_4000_tokens": 0.8116, + "mean_recall_at_16000_tokens": 0.8116, + "mean_precision_at_10": 0.1304, + "zero_hit_rate": 0, + "median_tokens_delivered": 3845, + "median_latency_ms": 708.038, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3758, + "full_recall_at_5": 0.1132, + "mean_recall_at_10": 0.5566, + "full_recall_at_10": 0.3396, + "mean_recall_at_20": 0.7343, + "full_recall_at_20": 0.5849, + "mean_recall_at_1000_tokens": 0.3664, + "mean_recall_at_4000_tokens": 0.7248, + "mean_recall_at_16000_tokens": 0.7343, + "mean_precision_at_10": 0.1038, + "zero_hit_rate": 0.1321, + "median_tokens_delivered": 3851, + "median_latency_ms": 701.209, + "payload_kind": [ + "chunks" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 71473.8, + "peak_rss_mb": 73218.7, + "delta_rss_mb": 1744.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codesearch", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.5099, + "full_recall_at_5": 0.3816, + "mean_recall_at_10": 0.5921, + "full_recall_at_10": 0.4737, + "mean_recall_at_20": 0.5921, + "full_recall_at_20": 0.4737, + "mean_recall_at_1000_tokens": 0.5099, + "mean_recall_at_4000_tokens": 0.5921, + "mean_recall_at_16000_tokens": 0.5921, + "mean_precision_at_10": 0.1692, + "zero_hit_rate": 0.2895, + "median_tokens_delivered": 1184, + "median_latency_ms": 884.899, + "payload_kind": [ + "chunks" + ], + "unavailable": 5 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.1632, + "full_recall_at_5": 0.0417, + "mean_recall_at_10": 0.2292, + "full_recall_at_10": 0.0833, + "mean_recall_at_20": 0.2292, + "full_recall_at_20": 0.0833, + "mean_recall_at_1000_tokens": 0.184, + "mean_recall_at_4000_tokens": 0.2292, + "mean_recall_at_16000_tokens": 0.2292, + "mean_precision_at_10": 0.0989, + "zero_hit_rate": 0.625, + "median_tokens_delivered": 1224, + "median_latency_ms": 868.714, + "payload_kind": [ + "chunks" + ], + "unavailable": 2 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.6699, + "full_recall_at_5": 0.5385, + "mean_recall_at_10": 0.7596, + "full_recall_at_10": 0.6538, + "mean_recall_at_20": 0.7596, + "full_recall_at_20": 0.6538, + "mean_recall_at_1000_tokens": 0.6603, + "mean_recall_at_4000_tokens": 0.7596, + "mean_recall_at_16000_tokens": 0.7596, + "mean_precision_at_10": 0.2016, + "zero_hit_rate": 0.1346, + "median_tokens_delivered": 1093.5, + "median_latency_ms": 885.586, + "payload_kind": [ + "chunks" + ], + "unavailable": 3 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.6377, + "full_recall_at_5": 0.4783, + "mean_recall_at_10": 0.8116, + "full_recall_at_10": 0.6957, + "mean_recall_at_20": 0.8116, + "full_recall_at_20": 0.6957, + "mean_recall_at_1000_tokens": 0.7319, + "mean_recall_at_4000_tokens": 0.8116, + "mean_recall_at_16000_tokens": 0.8116, + "mean_precision_at_10": 0.2379, + "zero_hit_rate": 0.087, + "median_tokens_delivered": 1051, + "median_latency_ms": 878.463, + "payload_kind": [ + "chunks" + ], + "unavailable": 1 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.4544, + "full_recall_at_5": 0.3396, + "mean_recall_at_10": 0.4969, + "full_recall_at_10": 0.3774, + "mean_recall_at_20": 0.4969, + "full_recall_at_20": 0.3774, + "mean_recall_at_1000_tokens": 0.4135, + "mean_recall_at_4000_tokens": 0.4969, + "mean_recall_at_16000_tokens": 0.4969, + "mean_precision_at_10": 0.1394, + "zero_hit_rate": 0.3774, + "median_tokens_delivered": 1227, + "median_latency_ms": 885.446, + "payload_kind": [ + "chunks" + ], + "unavailable": 4 + } + }, + "resources": { + "baseline_rss_mb": 67996.1, + "peak_rss_mb": 68634.9, + "delta_rss_mb": 638.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "graphify", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.4583, + "full_recall_at_5": 0.3026, + "mean_recall_at_10": 0.5417, + "full_recall_at_10": 0.4211, + "mean_recall_at_20": 0.5789, + "full_recall_at_20": 0.4342, + "mean_recall_at_1000_tokens": 0.4342, + "mean_recall_at_4000_tokens": 0.568, + "mean_recall_at_16000_tokens": 0.5789, + "mean_precision_at_10": 0.1672, + "zero_hit_rate": 0.2632, + "median_tokens_delivered": 1669, + "median_latency_ms": 924.396, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.2153, + "full_recall_at_5": 0.0833, + "mean_recall_at_10": 0.2639, + "full_recall_at_10": 0.1667, + "mean_recall_at_20": 0.3403, + "full_recall_at_20": 0.2083, + "mean_recall_at_1000_tokens": 0.1528, + "mean_recall_at_4000_tokens": 0.3264, + "mean_recall_at_16000_tokens": 0.3403, + "mean_precision_at_10": 0.1181, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 1690.5, + "median_latency_ms": 920.271, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.5705, + "full_recall_at_5": 0.4038, + "mean_recall_at_10": 0.6699, + "full_recall_at_10": 0.5385, + "mean_recall_at_20": 0.6891, + "full_recall_at_20": 0.5385, + "mean_recall_at_1000_tokens": 0.5641, + "mean_recall_at_4000_tokens": 0.6795, + "mean_recall_at_16000_tokens": 0.6891, + "mean_precision_at_10": 0.1898, + "zero_hit_rate": 0.1538, + "median_tokens_delivered": 1667.5, + "median_latency_ms": 928.2355, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.6196, + "full_recall_at_5": 0.4348, + "mean_recall_at_10": 0.7283, + "full_recall_at_10": 0.6087, + "mean_recall_at_20": 0.7283, + "full_recall_at_20": 0.6087, + "mean_recall_at_1000_tokens": 0.7065, + "mean_recall_at_4000_tokens": 0.7283, + "mean_recall_at_16000_tokens": 0.7283, + "mean_precision_at_10": 0.1979, + "zero_hit_rate": 0.1304, + "median_tokens_delivered": 1664, + "median_latency_ms": 885.322, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3884, + "full_recall_at_5": 0.2453, + "mean_recall_at_10": 0.4607, + "full_recall_at_10": 0.3396, + "mean_recall_at_20": 0.5142, + "full_recall_at_20": 0.3585, + "mean_recall_at_1000_tokens": 0.316, + "mean_recall_at_4000_tokens": 0.4984, + "mean_recall_at_16000_tokens": 0.5142, + "mean_precision_at_10": 0.1538, + "zero_hit_rate": 0.3208, + "median_tokens_delivered": 1674, + "median_latency_ms": 934.489, + "payload_kind": [ + "node-summaries" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 67285.2, + "peak_rss_mb": 71236, + "delta_rss_mb": 3950.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 117551104 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.3783, + "full_recall_at_5": 0.2632, + "mean_recall_at_10": 0.5077, + "full_recall_at_10": 0.3553, + "mean_recall_at_20": 0.7368, + "full_recall_at_20": 0.6579, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0175, + "mean_recall_at_16000_tokens": 0.2884, + "mean_precision_at_10": 0.0934, + "zero_hit_rate": 0.1842, + "median_tokens_delivered": 88454.5, + "median_latency_ms": 21.679, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.2049, + "full_recall_at_5": 0.0833, + "mean_recall_at_10": 0.2882, + "full_recall_at_10": 0.125, + "mean_recall_at_20": 0.4653, + "full_recall_at_20": 0.3333, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0139, + "mean_recall_at_16000_tokens": 0.1215, + "mean_precision_at_10": 0.0625, + "zero_hit_rate": 0.4167, + "median_tokens_delivered": 88074, + "median_latency_ms": 21.9125, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.4583, + "full_recall_at_5": 0.3462, + "mean_recall_at_10": 0.609, + "full_recall_at_10": 0.4615, + "mean_recall_at_20": 0.8622, + "full_recall_at_20": 0.8077, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0192, + "mean_recall_at_16000_tokens": 0.3654, + "mean_precision_at_10": 0.1077, + "zero_hit_rate": 0.0769, + "median_tokens_delivered": 89967, + "median_latency_ms": 21.501, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.4203, + "full_recall_at_5": 0.3478, + "mean_recall_at_10": 0.5507, + "full_recall_at_10": 0.3913, + "mean_recall_at_20": 0.7174, + "full_recall_at_20": 0.6087, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0435, + "mean_recall_at_16000_tokens": 0.3768, + "mean_precision_at_10": 0.0913, + "zero_hit_rate": 0.1739, + "median_tokens_delivered": 90059, + "median_latency_ms": 21.054, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3601, + "full_recall_at_5": 0.2264, + "mean_recall_at_10": 0.489, + "full_recall_at_10": 0.3396, + "mean_recall_at_20": 0.7453, + "full_recall_at_20": 0.6792, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0063, + "mean_recall_at_16000_tokens": 0.25, + "mean_precision_at_10": 0.0943, + "zero_hit_rate": 0.1887, + "median_tokens_delivered": 88304, + "median_latency_ms": 21.929, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 68109.7, + "peak_rss_mb": 68132.6, + "delta_rss_mb": 22.8, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.068, + "full_recall_at_5": 0.0132, + "mean_recall_at_10": 0.0921, + "full_recall_at_10": 0.0263, + "mean_recall_at_20": 0.1776, + "full_recall_at_20": 0.0526, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0263, + "mean_recall_at_16000_tokens": 0.1009, + "mean_precision_at_10": 0.0197, + "zero_hit_rate": 0.6842, + "median_tokens_delivered": 24792.5, + "median_latency_ms": 10.9515, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 24, + "mean_recall_at_5": 0.0347, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0486, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1319, + "full_recall_at_20": 0, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0069, + "mean_recall_at_16000_tokens": 0.0764, + "mean_precision_at_10": 0.0167, + "zero_hit_rate": 0.7083, + "median_tokens_delivered": 25739, + "median_latency_ms": 11.036, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 52, + "mean_recall_at_5": 0.0833, + "full_recall_at_5": 0.0192, + "mean_recall_at_10": 0.1122, + "full_recall_at_10": 0.0385, + "mean_recall_at_20": 0.1987, + "full_recall_at_20": 0.0769, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0353, + "mean_recall_at_16000_tokens": 0.1122, + "mean_precision_at_10": 0.0212, + "zero_hit_rate": 0.6731, + "median_tokens_delivered": 24792.5, + "median_latency_ms": 10.89, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.0435, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0652, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1522, + "full_recall_at_20": 0.0435, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0217, + "mean_recall_at_16000_tokens": 0.0652, + "mean_precision_at_10": 0.013, + "zero_hit_rate": 0.7391, + "median_tokens_delivered": 24180, + "median_latency_ms": 10.685, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.0786, + "full_recall_at_5": 0.0189, + "mean_recall_at_10": 0.1038, + "full_recall_at_10": 0.0377, + "mean_recall_at_20": 0.1887, + "full_recall_at_20": 0.0566, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0283, + "mean_recall_at_16000_tokens": 0.1164, + "mean_precision_at_10": 0.0226, + "zero_hit_rate": 0.6604, + "median_tokens_delivered": 25977, + "median_latency_ms": 11.107, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "resources": { + "baseline_rss_mb": 68186.3, + "peak_rss_mb": 68199.3, + "delta_rss_mb": 12.9, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "agent-default", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 59414, + "latency_ms": 66.603, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 71198, + "latency_ms": 53.71, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 76878, + "latency_ms": 57.748, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 57200, + "latency_ms": 48.98, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 25619, + "latency_ms": 41.564, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 67883, + "latency_ms": 43.402, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 33.219, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 73836, + "latency_ms": 62.53, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2441, + "latency_ms": 40.805, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 71759, + "latency_ms": 54.843, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 69187, + "latency_ms": 54.441, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 78746, + "latency_ms": 62.824, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 53332, + "latency_ms": 41.704, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 74381, + "latency_ms": 57.508, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 74795, + "latency_ms": 67.554, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 70497, + "latency_ms": 56.273, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13961, + "latency_ms": 47.51, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 79122, + "latency_ms": 36.532, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2143, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 71092, + "latency_ms": 50.029, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 66658, + "latency_ms": 43.181, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 20991, + "latency_ms": 56.091, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86943, + "latency_ms": 53.447, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 95744, + "latency_ms": 76.477, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 84785, + "latency_ms": 55.128, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67946, + "latency_ms": 40.029, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 67888, + "latency_ms": 45.993, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 47664, + "latency_ms": 52.365, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.8333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 61019, + "latency_ms": 42.413, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "gin.go", + "test_helpers.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 67859, + "latency_ms": 68.127, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 79884, + "latency_ms": 45.169, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 56517, + "latency_ms": 59.958, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 94416, + "latency_ms": 66.995, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 86766, + "latency_ms": 69.108, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 98524, + "latency_ms": 62.491, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 64060, + "latency_ms": 69.479, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 66751, + "latency_ms": 54.311, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 48437, + "latency_ms": 54.792, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 53447, + "latency_ms": 39.586, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 344, + "latency_ms": 40.153, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 76691, + "latency_ms": 55.783, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 63498, + "latency_ms": 53.534, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 79093, + "latency_ms": 54.796, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 60197, + "latency_ms": 39.953, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 59600, + "latency_ms": 48.745, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 62885, + "latency_ms": 44.76, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 72047, + "latency_ms": 56.582, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 76351, + "latency_ms": 67.961, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 66011, + "latency_ms": 73.613, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 21105, + "latency_ms": 40.178, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 58056, + "latency_ms": 42.307, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 90063, + "latency_ms": 62.044, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "recovery_test.go" + ], + "missed": [ + "errors.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 102004, + "latency_ms": 78.26, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 77237, + "latency_ms": 54.262, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 54210, + "latency_ms": 54.141, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 88221, + "latency_ms": 69.685, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 39288, + "latency_ms": 41.16, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47970, + "latency_ms": 41.072, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 58227, + "latency_ms": 53.987, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go" + ], + "missed": [ + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 92238, + "latency_ms": 60.22, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 103343, + "latency_ms": 42.632, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 64119, + "latency_ms": 47.514, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 33.271, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 65504, + "latency_ms": 74.722, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 87536, + "latency_ms": 64.764, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80910, + "latency_ms": 47.38, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 61055, + "latency_ms": 69.104, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 61558, + "latency_ms": 48.772, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 92311, + "latency_ms": 63.78, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 104156, + "latency_ms": 78.779, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 81488, + "latency_ms": 55.818, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 71433, + "latency_ms": 71.144, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 84500, + "latency_ms": 86.625, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 21802, + "latency_ms": 56.313, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 78640, + "latency_ms": 71.443, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 52456, + "latency_ms": 46.996, + "indexed_revision_matches": true + }, + { + "provider_id": "agent-default", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 52554, + "latency_ms": 41.686, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 60149, + "latency_ms": 91.094, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 71198, + "latency_ms": 90.049, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 84027, + "latency_ms": 88.252, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 50548, + "latency_ms": 85.685, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 25619, + "latency_ms": 73.385, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 67883, + "latency_ms": 81.97, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 41.522, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 73836, + "latency_ms": 103.837, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2441, + "latency_ms": 50.731, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 75648, + "latency_ms": 96.835, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 69187, + "latency_ms": 87.121, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 78746, + "latency_ms": 104.259, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 53332, + "latency_ms": 61.246, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 74381, + "latency_ms": 103.937, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 86796, + "latency_ms": 107.655, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 70497, + "latency_ms": 100.407, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13961, + "latency_ms": 51.495, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 79122, + "latency_ms": 76.127, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2143, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 71092, + "latency_ms": 77.534, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 66658, + "latency_ms": 85.398, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 53169, + "latency_ms": 95.662, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86996, + "latency_ms": 97.414, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 98248, + "latency_ms": 116.968, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 84785, + "latency_ms": 96.487, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67946, + "latency_ms": 68.085, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 68660, + "latency_ms": 85.541, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 47664, + "latency_ms": 77.993, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.8333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 61019, + "latency_ms": 82.104, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "test_helpers.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 73530, + "latency_ms": 107.41, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 79884, + "latency_ms": 80.484, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 58532, + "latency_ms": 96.995, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 94416, + "latency_ms": 104.072, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 86766, + "latency_ms": 105.257, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 98524, + "latency_ms": 105.163, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 69644, + "latency_ms": 111.535, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 90825, + "latency_ms": 91.819, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 53828, + "latency_ms": 88.228, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 55487, + "latency_ms": 73.88, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 344, + "latency_ms": 46.455, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 76691, + "latency_ms": 101.041, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode_test.go" + ], + "missed": [ + "mode.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 65844, + "latency_ms": 97.269, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 79093, + "latency_ms": 104.068, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62345, + "latency_ms": 82.367, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 59600, + "latency_ms": 111.433, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 62885, + "latency_ms": 86.957, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 73065, + "latency_ms": 98.603, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 84515, + "latency_ms": 110.371, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 68823, + "latency_ms": 109.81, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 64033, + "latency_ms": 79.577, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 58056, + "latency_ms": 63.317, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93773, + "latency_ms": 104.163, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "recovery_test.go" + ], + "missed": [ + "errors.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 102004, + "latency_ms": 127.131, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 77237, + "latency_ms": 99.099, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 58472, + "latency_ms": 102.116, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93890, + "latency_ms": 114.24, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 39288, + "latency_ms": 58.158, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47970, + "latency_ms": 81.053, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 59839, + "latency_ms": 96.282, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go" + ], + "missed": [ + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 92238, + "latency_ms": 117.233, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 103343, + "latency_ms": 86.039, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 69750, + "latency_ms": 89.511, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 40.574, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 87717, + "latency_ms": 115.848, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 87368, + "latency_ms": 102.169, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80910, + "latency_ms": 81.854, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 61055, + "latency_ms": 90.607, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 61558, + "latency_ms": 84.98, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 92311, + "latency_ms": 105.721, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 104156, + "latency_ms": 113.504, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82299, + "latency_ms": 88.592, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 71447, + "latency_ms": 104.318, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 84500, + "latency_ms": 119.69, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 70865, + "latency_ms": 96.195, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 78640, + "latency_ms": 285.523, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 56727, + "latency_ms": 246.168, + "indexed_revision_matches": true + }, + { + "provider_id": "ripgrep", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 64337, + "latency_ms": 87.214, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 60149, + "latency_ms": 62.214, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 71198, + "latency_ms": 42.73, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 84027, + "latency_ms": 40.037, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 50548, + "latency_ms": 31.229, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 25619, + "latency_ms": 32.017, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 67883, + "latency_ms": 112.367, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 49.192, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 73836, + "latency_ms": 51.911, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2441, + "latency_ms": 30.997, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 75648, + "latency_ms": 36.473, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 69187, + "latency_ms": 40.995, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 78746, + "latency_ms": 54.047, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 53332, + "latency_ms": 20.405, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 74381, + "latency_ms": 42.125, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 86796, + "latency_ms": 49.282, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 70497, + "latency_ms": 40.007, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13961, + "latency_ms": 25.145, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 79122, + "latency_ms": 19.24, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2143, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 71092, + "latency_ms": 36.772, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 66658, + "latency_ms": 26.119, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 53169, + "latency_ms": 33.692, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86996, + "latency_ms": 33.13, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 98248, + "latency_ms": 53.556, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 84785, + "latency_ms": 34.73, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 67946, + "latency_ms": 27.196, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 68660, + "latency_ms": 27.974, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 47664, + "latency_ms": 33.086, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.6, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.8333, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 61019, + "latency_ms": 26.279, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "test_helpers.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 73530, + "latency_ms": 47.667, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 79884, + "latency_ms": 28.547, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 58532, + "latency_ms": 41.259, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 94416, + "latency_ms": 45.916, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 86766, + "latency_ms": 52.899, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 98524, + "latency_ms": 44.299, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 69644, + "latency_ms": 51.126, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 90825, + "latency_ms": 32.052, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 53828, + "latency_ms": 34.911, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 55487, + "latency_ms": 19.977, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 344, + "latency_ms": 18.832, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 76691, + "latency_ms": 44.997, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode_test.go" + ], + "missed": [ + "mode.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 65844, + "latency_ms": 32.8, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 79093, + "latency_ms": 41.313, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62345, + "latency_ms": 19.873, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 59600, + "latency_ms": 35.198, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 62885, + "latency_ms": 25.476, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 73065, + "latency_ms": 33.121, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 84515, + "latency_ms": 51.339, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 68823, + "latency_ms": 53.078, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 64033, + "latency_ms": 19.862, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 58056, + "latency_ms": 19.269, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 93773, + "latency_ms": 40.351, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "recovery_test.go" + ], + "missed": [ + "errors.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 3, + "tokens_delivered": 102004, + "latency_ms": 64.32, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 77237, + "latency_ms": 34.556, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 58472, + "latency_ms": 37.58, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93890, + "latency_ms": 55.038, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 39288, + "latency_ms": 26.123, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 47970, + "latency_ms": 32.982, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 59839, + "latency_ms": 40.469, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go" + ], + "missed": [ + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 92238, + "latency_ms": 50.74, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 103343, + "latency_ms": 30.009, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 69750, + "latency_ms": 26.901, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 21.175, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 87717, + "latency_ms": 58.115, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 87368, + "latency_ms": 43.113, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80910, + "latency_ms": 28.086, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 61055, + "latency_ms": 50.69, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 61558, + "latency_ms": 37.3, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 92311, + "latency_ms": 49.151, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 104156, + "latency_ms": 56.757, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 82299, + "latency_ms": 33.622, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 71447, + "latency_ms": 44.524, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 84500, + "latency_ms": 64.542, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 70865, + "latency_ms": 35.36, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 78640, + "latency_ms": 55.204, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 56727, + "latency_ms": 28.28, + "indexed_revision_matches": true + }, + { + "provider_id": "keyword-search", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 64337, + "latency_ms": 22.018, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 160, + "latency_ms": 10.911, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.392, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 30452, + "latency_ms": 11.721, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 57200, + "latency_ms": 11.233, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.975, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.667, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 6.048, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2559, + "latency_ms": 11.353, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.728, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 10147, + "latency_ms": 10.493, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1664, + "latency_ms": 11.162, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2061, + "latency_ms": 10.906, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63, + "latency_ms": 11.147, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.451, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 28859, + "latency_ms": 10.597, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.066, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.422, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.294, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.139, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.403, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 20991, + "latency_ms": 10.129, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 115, + "latency_ms": 10.955, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2386, + "latency_ms": 10.176, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2061, + "latency_ms": 10.035, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.355, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 218, + "latency_ms": 10.426, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 494, + "latency_ms": 11.782, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.57, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "gin.go", + "test_helpers.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 4, + "tokens_delivered": 55223, + "latency_ms": 12.007, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63, + "latency_ms": 12.044, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 9403, + "latency_ms": 12.066, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6194, + "latency_ms": 13.193, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13845, + "latency_ms": 13.897, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 6.859, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 28809, + "latency_ms": 13.228, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 25055, + "latency_ms": 13.797, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 26350, + "latency_ms": 15.729, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 3, + "tokens_delivered": 23874, + "latency_ms": 12.489, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 1, + "recall_at_10": 0.25, + "precision_at_10": 1, + "recall_at_20": 0.25, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 174, + "latency_ms": 11.407, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.696, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "mode.go", + "mode_test.go" + ], + "missed": [ + "debug.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 1173, + "latency_ms": 11.025, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.386, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1593, + "latency_ms": 11.087, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.904, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.837, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 218, + "latency_ms": 11.512, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35535, + "latency_ms": 10.785, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [], + "missed": [ + "gin.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1554, + "latency_ms": 10.517, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 21105, + "latency_ms": 11.288, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 226, + "latency_ms": 10.384, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 273, + "latency_ms": 10.828, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.13, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 13341, + "latency_ms": 11.899, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 28275, + "latency_ms": 11.06, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 217, + "latency_ms": 11.126, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.604, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.129, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 52092, + "latency_ms": 11.75, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.091, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.393, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 28495, + "latency_ms": 10.9, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.623, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 26196, + "latency_ms": 11.486, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [], + "missed": [ + "gin.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 234, + "latency_ms": 11.171, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63, + "latency_ms": 10.074, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 6.403, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.587, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 6.643, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 234, + "latency_ms": 11.008, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 115, + "latency_ms": 10.862, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 507, + "latency_ms": 10.579, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.522, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 20618, + "latency_ms": 10.46, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 5.865, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63, + "latency_ms": 10.684, + "indexed_revision_matches": true + }, + { + "provider_id": "filename-match", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 54794, + "latency_ms": 11.024, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go", + "logger_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 28336, + "latency_ms": 346.566, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 67144, + "latency_ms": 263.403, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 61805, + "latency_ms": 244.88, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 55582, + "latency_ms": 204.82, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 25619, + "latency_ms": 211.423, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 59487, + "latency_ms": 199.498, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 168.603, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 28018, + "latency_ms": 285.121, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.3333, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4270, + "latency_ms": 199.388, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 36990, + "latency_ms": 250.461, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 73040, + "latency_ms": 245.719, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 88275, + "latency_ms": 285.879, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 53332, + "latency_ms": 171.609, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 69963, + "latency_ms": 290.756, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 59249, + "latency_ms": 333.096, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 32550, + "latency_ms": 281.469, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13961, + "latency_ms": 200.595, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 74578, + "latency_ms": 171.668, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2143, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 71092, + "latency_ms": 253.794, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 66844, + "latency_ms": 205.624, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 12, + "tokens_delivered": 8229, + "latency_ms": 259.748, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 77983, + "latency_ms": 245.156, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 90938, + "latency_ms": 375.495, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 75178, + "latency_ms": 254.728, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70230, + "latency_ms": 208.288, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 74301, + "latency_ms": 212.175, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 50383, + "latency_ms": 248.532, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.1667, + "first_hit_rank": 1, + "tokens_delivered": 67487, + "latency_ms": 214.83, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 56088, + "latency_ms": 350.752, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 62255, + "latency_ms": 214.994, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go" + ], + "missed": [ + "render/render_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 31321, + "latency_ms": 312.045, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 92189, + "latency_ms": 341.139, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 83666, + "latency_ms": 342.445, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 95800, + "latency_ms": 357.64, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 67140, + "latency_ms": 360.351, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 13, + "tokens_delivered": 8982, + "latency_ms": 280.482, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 55153, + "latency_ms": 267.61, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 54954, + "latency_ms": 181.512, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 34473, + "latency_ms": 178.626, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 65043, + "latency_ms": 330.27, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23611, + "latency_ms": 269.816, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 79681, + "latency_ms": 323.537, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 18, + "tokens_delivered": 34979, + "latency_ms": 185.076, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 30910, + "latency_ms": 320.466, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 65364, + "latency_ms": 234.673, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 80590, + "latency_ms": 303.54, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 75420, + "latency_ms": 385.354, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 60032, + "latency_ms": 375.852, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 8292, + "latency_ms": 197.405, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 62207, + "latency_ms": 201.241, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 90648, + "latency_ms": 333.037, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "errors.go" + ], + "missed": [ + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 36444, + "latency_ms": 503.497, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 68191, + "latency_ms": 288.646, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 53700, + "latency_ms": 277.613, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 84784, + "latency_ms": 385.851, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 39288, + "latency_ms": 243.609, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49907, + "latency_ms": 220.862, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26027, + "latency_ms": 326.946, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 73755, + "latency_ms": 389.935, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 95021, + "latency_ms": 275.547, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 57641, + "latency_ms": 249.937, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 181.299, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 77479, + "latency_ms": 457.964, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 89708, + "latency_ms": 345.844, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 84777, + "latency_ms": 246.098, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 68527, + "latency_ms": 368.263, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 63311, + "latency_ms": 296.352, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 102476, + "latency_ms": 378.181, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 73727, + "latency_ms": 438.211, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 67521, + "latency_ms": 313.301, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 45425, + "latency_ms": 405.964, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 31293, + "latency_ms": 517.737, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 18228, + "latency_ms": 347.484, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 90051, + "latency_ms": 454.678, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 23460, + "latency_ms": 272.693, + "indexed_revision_matches": true + }, + { + "provider_id": "zoekt", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 55996, + "latency_ms": 259.021, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [ + "logger_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.6667, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 44618, + "latency_ms": 236.484, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 62533, + "latency_ms": 239.969, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 68824, + "latency_ms": 216.021, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20161, + "latency_ms": 177.671, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 189.22, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2868, + "latency_ms": 172.601, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 121.513, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 5799, + "latency_ms": 297.651, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 173.74, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 51252, + "latency_ms": 221.057, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 66999, + "latency_ms": 311.182, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 41424, + "latency_ms": 289.784, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 143.166, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 73733, + "latency_ms": 278.301, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 69439, + "latency_ms": 314.343, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 16163, + "latency_ms": 274.854, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 201.083, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3616, + "latency_ms": 145.052, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.3333, + "recall_at_20": 0.6667, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 3, + "tokens_delivered": 30672, + "latency_ms": 252.8, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4333, + "latency_ms": 190.472, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 41184, + "latency_ms": 254.429, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30672, + "latency_ms": 240.505, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 88499, + "latency_ms": 408.637, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 44669, + "latency_ms": 234.413, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 166.982, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 7587, + "latency_ms": 175.673, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 374, + "latency_ms": 221.961, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "response_writer.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 19019, + "latency_ms": 174.185, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "test_helpers.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 66348, + "latency_ms": 323.221, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 172.342, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 33240, + "latency_ms": 286.357, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 71592, + "latency_ms": 345.42, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 77949, + "latency_ms": 320.981, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 54350, + "latency_ms": 312.539, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 69321, + "latency_ms": 321.496, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 82883, + "latency_ms": 226.641, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24462, + "latency_ms": 248.831, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1318, + "latency_ms": 125.437, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 1, + "recall_at_10": 0.25, + "precision_at_10": 1, + "recall_at_20": 0.25, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 174, + "latency_ms": 120.258, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 58698, + "latency_ms": 271.125, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "mode.go" + ], + "missed": [ + "debug.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 63126, + "latency_ms": 228.73, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 32451, + "latency_ms": 265.705, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28177, + "latency_ms": 137.724, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21327, + "latency_ms": 228.402, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go" + ], + "missed": [ + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 29892, + "latency_ms": 171.26, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 43729, + "latency_ms": 231.575, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 40488, + "latency_ms": 325.43, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 33577, + "latency_ms": 326.165, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 29638, + "latency_ms": 117.513, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1306, + "latency_ms": 128.735, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 9568, + "latency_ms": 289.909, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1667, + "recall_at_20": 0.25, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 52734, + "latency_ms": 427.375, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 19030, + "latency_ms": 231.708, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 53053, + "latency_ms": 223.949, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 9, + "tokens_delivered": 65277, + "latency_ms": 343.237, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 6614, + "latency_ms": 168.074, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 11311, + "latency_ms": 181.053, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19289, + "latency_ms": 256.819, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go" + ], + "missed": [ + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 51861, + "latency_ms": 346.397, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 17, + "tokens_delivered": 79685, + "latency_ms": 205.672, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 2, + "tokens_delivered": 27658, + "latency_ms": 183.188, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 116.764, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 44550, + "latency_ms": 389.355, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 51324, + "latency_ms": 290.403, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 188.701, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 315.079, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 11049, + "latency_ms": 239.394, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 59515, + "latency_ms": 331.436, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1111, + "recall_at_20": 0.5, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 27416, + "latency_ms": 414.631, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 15350, + "latency_ms": 221.864, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 381, + "latency_ms": 345.062, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 64058, + "latency_ms": 428.155, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44432, + "latency_ms": 226.749, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 388.185, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 176.512, + "indexed_revision_matches": true + }, + { + "provider_id": "ast-grep", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19624, + "latency_ms": 122.789, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [ + "logger_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3949, + "latency_ms": 612.458, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "debug_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3413, + "latency_ms": 635.049, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4196, + "latency_ms": 631.556, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3356, + "latency_ms": 615.962, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3341, + "latency_ms": 659.74, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3915, + "latency_ms": 617.337, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 17, + "tokens_delivered": 3102, + "latency_ms": 624.858, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3950, + "latency_ms": 708.038, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3594, + "latency_ms": 656.615, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3675, + "latency_ms": 648.956, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4148, + "latency_ms": 543.588, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4239, + "latency_ms": 690.412, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3553, + "latency_ms": 643.683, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3938, + "latency_ms": 598.732, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3845, + "latency_ms": 586.99, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3792, + "latency_ms": 586.877, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [ + "fs.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 19, + "tokens_delivered": 3517, + "latency_ms": 552.21, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 4318, + "latency_ms": 616.199, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.1765, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3808, + "latency_ms": 643.522, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3942, + "latency_ms": 592.517, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 3939, + "latency_ms": 673.914, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 4008, + "latency_ms": 619.304, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3618, + "latency_ms": 700.751, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3892, + "latency_ms": 749.811, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3630, + "latency_ms": 597.938, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 3801, + "latency_ms": 564.096, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3704, + "latency_ms": 639.833, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "errors.go", + "gin.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "context_test.go", + "response_writer.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.1667, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 3726, + "latency_ms": 670.389, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.2143, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3829, + "latency_ms": 1206.633, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 9, + "tokens_delivered": 3720, + "latency_ms": 951.618, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go" + ], + "missed": [ + "render/render_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3768, + "latency_ms": 683.918, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4121, + "latency_ms": 708.198, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3904, + "latency_ms": 705.305, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4266, + "latency_ms": 775.493, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4026, + "latency_ms": 772.822, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3749, + "latency_ms": 720.514, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4030, + "latency_ms": 665.421, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3415, + "latency_ms": 646.227, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3930, + "latency_ms": 734.627, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3715, + "latency_ms": 701.209, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go" + ], + "missed": [ + "mode_test.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 3663, + "latency_ms": 710.681, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4155, + "latency_ms": 654.86, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3024, + "latency_ms": 729.415, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 3276, + "latency_ms": 702.424, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go" + ], + "missed": [ + "binding/validate_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4094, + "latency_ms": 707.506, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 9, + "tokens_delivered": 4139, + "latency_ms": 695.123, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4014, + "latency_ms": 745.594, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4052, + "latency_ms": 665.543, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4032, + "latency_ms": 684.302, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3851, + "latency_ms": 694.611, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4035, + "latency_ms": 771.393, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "errors.go", + "routergroup.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.75, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.75, + "recall_at_16000_tokens": 0.75, + "first_hit_rank": 4, + "tokens_delivered": 3942, + "latency_ms": 788.788, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3626, + "latency_ms": 853.956, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3949, + "latency_ms": 845.181, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 3891, + "latency_ms": 876.644, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3804, + "latency_ms": 810.693, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3933, + "latency_ms": 813.046, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 3626, + "latency_ms": 837.472, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go" + ], + "missed": [ + "errors_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 4241, + "latency_ms": 801.711, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 3656, + "latency_ms": 839.783, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3696, + "latency_ms": 878.029, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [ + "recovery.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 16, + "tokens_delivered": 3949, + "latency_ms": 834.653, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 3892, + "latency_ms": 889.061, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4230, + "latency_ms": 967.21, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3704, + "latency_ms": 774.413, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 16, + "tokens_delivered": 3800, + "latency_ms": 810.32, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3741, + "latency_ms": 789.034, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4189, + "latency_ms": 774.795, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3931, + "latency_ms": 885.389, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3774, + "latency_ms": 791.939, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3918, + "latency_ms": 763.407, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3780, + "latency_ms": 878.361, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3870, + "latency_ms": 701.679, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3844, + "latency_ms": 753.193, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "debug_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3236, + "latency_ms": 678.993, + "indexed_revision_matches": true + }, + { + "provider_id": "semble", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 3146, + "latency_ms": 680.06, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4742, + "latency_ms": 1069.267, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 518, + "latency_ms": 853.043, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 914, + "latency_ms": 841.566, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1339, + "latency_ms": 771.562, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2062, + "latency_ms": 810.682, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1027, + "latency_ms": 786.444, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 483, + "latency_ms": 823.101, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1509, + "latency_ms": 1045.982, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 852.259, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 897, + "latency_ms": 1058.787, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1034, + "latency_ms": 797.808, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1188, + "latency_ms": 921.548, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1368, + "latency_ms": 976.406, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 722, + "latency_ms": 815.739, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1104, + "latency_ms": 784.981, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1538, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 974, + "latency_ms": 810.707, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1227, + "latency_ms": 750.778, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 693, + "latency_ms": 926.391, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.3, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3585, + "latency_ms": 804.487, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 976, + "latency_ms": 768.742, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 766, + "latency_ms": 818.895, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1313, + "latency_ms": 807.231, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2780, + "latency_ms": 893.745, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1180, + "latency_ms": 885.446, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1275, + "latency_ms": 838.644, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "routes_test.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 515, + "latency_ms": 854.202, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 797, + "latency_ms": 989.762, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "binding/default_validator.go", + "gin.go" + ], + "missed": [ + "context_test.go", + "errors.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 2517, + "latency_ms": 849.954, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "test_helpers.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2857, + "recall_at_20": 0.6667, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 1309, + "latency_ms": 851.982, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2658, + "latency_ms": 870.86, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1045, + "latency_ms": 1420.263, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 740, + "latency_ms": 1168.042, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1841, + "latency_ms": 989.137, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1078, + "latency_ms": 923.452, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 772, + "latency_ms": 860.055, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1323, + "latency_ms": 857.883, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 851.326, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 501, + "latency_ms": 805.072, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/json.go", + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.4, + "recall_at_10": 0.5, + "precision_at_10": 0.2857, + "recall_at_20": 0.5, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1051, + "latency_ms": 842.223, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 834.483, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.3, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1083, + "latency_ms": 844.935, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3122, + "latency_ms": 884.352, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 832.888, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 478, + "latency_ms": 948.329, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 800, + "latency_ms": 926.445, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 758, + "latency_ms": 761.396, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 982, + "latency_ms": 813.867, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2592, + "latency_ms": 805.463, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 643, + "latency_ms": 917.542, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1261, + "latency_ms": 886.82, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1574, + "latency_ms": 938.133, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "errors.go" + ], + "missed": [ + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2857, + "recall_at_20": 0.5, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 1750, + "latency_ms": 947.261, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4044, + "latency_ms": 921.811, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 649, + "latency_ms": 878.463, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 4069, + "latency_ms": 1010.678, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 3216, + "latency_ms": 899.093, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1358, + "latency_ms": 1016.581, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1362, + "latency_ms": 1081.659, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [ + "errors.go" + ], + "missed": [ + "errors_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 765, + "latency_ms": 892.715, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 897, + "latency_ms": 1005.201, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 546, + "latency_ms": 911.217, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1387, + "latency_ms": 888.627, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2075, + "latency_ms": 1018.337, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2284, + "latency_ms": 1014.918, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1221, + "latency_ms": 843.96, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 805.389, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1967, + "latency_ms": 803.129, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 750, + "latency_ms": 951.002, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2857, + "recall_at_20": 1, + "precision_at_20": 0.2857, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1771, + "latency_ms": 982.047, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2505, + "latency_ms": 1091.897, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0588, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 2502, + "latency_ms": 957.15, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1921, + "latency_ms": 1079.172, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/protobuf.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.125, + "recall_at_20": 0.5, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 925, + "latency_ms": 897.688, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1237, + "latency_ms": 916.332, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 7, + "tokens_delivered": 1338, + "latency_ms": 821.911, + "indexed_revision_matches": true + }, + { + "provider_id": "codesearch", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1111, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 1295, + "latency_ms": 903.818, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [ + "logger_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 5, + "tokens_delivered": 6018, + "latency_ms": 1001.901, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 1674, + "latency_ms": 887.832, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1662, + "latency_ms": 942.318, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1664, + "latency_ms": 925.162, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 6022, + "latency_ms": 1024.961, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 7862, + "latency_ms": 906.849, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4075, + "latency_ms": 962.935, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1671, + "latency_ms": 1128.625, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7, + "latency_ms": 897.949, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 221, + "latency_ms": 1082.487, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 666, + "latency_ms": 931.309, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 190, + "latency_ms": 1124.554, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 80, + "latency_ms": 1130.116, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1667, + "latency_ms": 997.455, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1658, + "latency_ms": 1093.254, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4145, + "latency_ms": 980.26, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8733, + "latency_ms": 977.924, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 10, + "tokens_delivered": 1672, + "latency_ms": 1611.504, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go", + "tree_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 1, + "recall_at_10": 0.3333, + "precision_at_10": 1, + "recall_at_20": 0.3333, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 1, + "tokens_delivered": 159, + "latency_ms": 1131.837, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1676, + "latency_ms": 1004.942, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 4283, + "latency_ms": 1029.307, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 494, + "latency_ms": 1056.924, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1661, + "latency_ms": 1223.929, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 1675, + "latency_ms": 1231.191, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 2269, + "latency_ms": 1025.345, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1663, + "latency_ms": 1044.033, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2061, + "latency_ms": 1098.465, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go" + ], + "recall_at_5": 0.1667, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.3, + "recall_at_20": 0.5, + "precision_at_20": 0.3, + "recall_at_1000_tokens": 0.1667, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 1665, + "latency_ms": 1055.923, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.1579, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2706, + "latency_ms": 992.346, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 483, + "latency_ms": 1045.027, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/data.go", + "render/render_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1662, + "latency_ms": 1065.797, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1669, + "latency_ms": 1136, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1669, + "latency_ms": 934.489, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1673, + "latency_ms": 1138.141, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0909, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2333, + "latency_ms": 915.997, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8411, + "latency_ms": 885.322, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1664, + "latency_ms": 822.796, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1667, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1654, + "latency_ms": 768.517, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 1, + "recall_at_10": 0.25, + "precision_at_10": 1, + "recall_at_20": 0.25, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 128, + "latency_ms": 947.201, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1660, + "latency_ms": 811.669, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.3, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1668, + "latency_ms": 833.33, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 144, + "latency_ms": 798.586, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 19, + "tokens_delivered": 1673, + "latency_ms": 786.339, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 5798, + "latency_ms": 777.831, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/validate_test.go" + ], + "missed": [ + "binding/default_validator.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 2608, + "latency_ms": 842.742, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1666, + "latency_ms": 819.97, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1137, + "latency_ms": 825.304, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1668, + "latency_ms": 808.955, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [ + "binding/query.go" + ], + "missed": [ + "binding/query_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 1038, + "latency_ms": 830.234, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 209, + "latency_ms": 844.287, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1705, + "latency_ms": 952.631, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.25, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 4, + "tokens_delivered": 3336, + "latency_ms": 923.63, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1662, + "latency_ms": 787.99, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1662, + "latency_ms": 817.143, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 6602, + "latency_ms": 892.984, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.6667, + "recall_at_10": 1, + "precision_at_10": 0.6667, + "recall_at_20": 1, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 4366, + "latency_ms": 795.927, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1676, + "latency_ms": 799.076, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 1666, + "latency_ms": 845.092, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1664, + "latency_ms": 838.28, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 6296, + "latency_ms": 911.532, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1429, + "recall_at_20": 0.5, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 1665, + "latency_ms": 814.566, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1666, + "latency_ms": 739.575, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 1681, + "latency_ms": 884.531, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1429, + "recall_at_20": 1, + "precision_at_20": 0.1429, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 6, + "tokens_delivered": 3782, + "latency_ms": 870.8, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 192, + "latency_ms": 798.461, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 3970, + "latency_ms": 848.149, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1176, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 5, + "tokens_delivered": 1686, + "latency_ms": 768.079, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1669, + "latency_ms": 940.136, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 5, + "tokens_delivered": 5273, + "latency_ms": 949.035, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 6577, + "latency_ms": 947.195, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.0526, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 1667, + "latency_ms": 982.932, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 5820, + "latency_ms": 950.711, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/protobuf.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1667, + "recall_at_20": 0.5, + "precision_at_20": 0.1667, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 7343, + "latency_ms": 868.298, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 7724, + "latency_ms": 916.912, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug_test.go" + ], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 20, + "tokens_delivered": 1679, + "latency_ms": 815.43, + "indexed_revision_matches": true + }, + { + "provider_id": "graphify", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1664, + "latency_ms": 884.17, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go", + "logger_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 190, + "latency_ms": 655.482, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34, + "latency_ms": 677.113, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 454, + "latency_ms": 742.981, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12, + "latency_ms": 676.252, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 209, + "latency_ms": 757.359, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63, + "latency_ms": 702.272, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 329, + "latency_ms": 734.233, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 23, + "latency_ms": 993.35, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3, + "latency_ms": 876.465, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 59, + "latency_ms": 794.722, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 205, + "latency_ms": 674.556, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 758, + "latency_ms": 988.414, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28, + "latency_ms": 916.39, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 762, + "latency_ms": 713.582, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1329, + "latency_ms": 738.261, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28, + "latency_ms": 693.637, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8756, + "latency_ms": 634.876, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2827, + "latency_ms": 783.972, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 13, + "latency_ms": 731.74, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3758, + "latency_ms": 669.877, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 126, + "latency_ms": 737.069, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54, + "latency_ms": 716.036, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 308, + "latency_ms": 905.288, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10, + "latency_ms": 960.996, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 717.879, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "routes_test.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 19, + "latency_ms": 723.547, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 30, + "latency_ms": 767.202, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 60, + "latency_ms": 709.606, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "test_helpers.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 6, + "tokens_delivered": 5359, + "latency_ms": 699.306, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55, + "latency_ms": 843.309, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 7, + "latency_ms": 930.72, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 67, + "latency_ms": 886.818, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1086, + "latency_ms": 770.126, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 92, + "latency_ms": 892.001, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 18, + "tokens_delivered": 806, + "latency_ms": 716.138, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 17, + "tokens_delivered": 581, + "latency_ms": 830.592, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2, + "latency_ms": 656.113, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1115, + "latency_ms": 599.902, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.5, + "recall_at_10": 0.25, + "precision_at_10": 0.5, + "recall_at_20": 0.25, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 35, + "latency_ms": 804.275, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 329, + "latency_ms": 696.758, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "mode.go", + "mode_test.go" + ], + "missed": [ + "debug.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 253, + "latency_ms": 716.063, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 120, + "latency_ms": 718.431, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 683.238, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 11, + "latency_ms": 650.382, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/validate_test.go" + ], + "missed": [ + "binding/default_validator.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 94, + "latency_ms": 704.943, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "routes_test.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 19, + "latency_ms": 693.871, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 29, + "latency_ms": 761.311, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 24, + "latency_ms": 636.866, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45, + "latency_ms": 691.147, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 903, + "latency_ms": 710.811, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 120, + "latency_ms": 916.14, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12193, + "latency_ms": 904.727, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 99, + "latency_ms": 668.155, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 1700, + "latency_ms": 663.224, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 23, + "latency_ms": 858.776, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 52, + "latency_ms": 666.797, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 78, + "latency_ms": 661.57, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 68, + "latency_ms": 678.803, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 994, + "latency_ms": 778.232, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 71, + "latency_ms": 859.006, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 289, + "latency_ms": 684.108, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 470, + "latency_ms": 620.924, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 5, + "latency_ms": 841.009, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 11, + "latency_ms": 845.327, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16, + "latency_ms": 709.401, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8, + "latency_ms": 649.152, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 56, + "latency_ms": 635.007, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 280, + "latency_ms": 885.321, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 5, + "latency_ms": 884.67, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4, + "latency_ms": 731.736, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 9, + "latency_ms": 743.824, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 856, + "latency_ms": 761.47, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "binding/protobuf.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 509, + "latency_ms": 647.212, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 160, + "latency_ms": 816.638, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13, + "latency_ms": 585.15, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8, + "latency_ms": 687.06, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 69810, + "latency_ms": 23.415, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 84576, + "latency_ms": 21.537, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87509, + "latency_ms": 20.416, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 82399, + "latency_ms": 22.723, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 83992, + "latency_ms": 21.893, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 72743, + "latency_ms": 22.524, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 88246, + "latency_ms": 23.975, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105412, + "latency_ms": 22.464, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 84449, + "latency_ms": 22.514, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 95317, + "latency_ms": 22.414, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 69979, + "latency_ms": 23.19, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 113276, + "latency_ms": 23.068, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 108509, + "latency_ms": 21.465, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 84446, + "latency_ms": 20.634, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 90304, + "latency_ms": 21.821, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 89875, + "latency_ms": 22.084, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70138, + "latency_ms": 22.282, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93950, + "latency_ms": 21.092, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 87314, + "latency_ms": 20.02, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82768, + "latency_ms": 19.669, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 88101, + "latency_ms": 20.895, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 87276, + "latency_ms": 21.004, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 117055, + "latency_ms": 22.287, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 113658, + "latency_ms": 22.242, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 87849, + "latency_ms": 22.904, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 89097, + "latency_ms": 23.646, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93972, + "latency_ms": 21.892, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 88010, + "latency_ms": 21.163, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "test_helpers.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 88138, + "latency_ms": 19.71, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 103341, + "latency_ms": 19.644, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 115951, + "latency_ms": 20.702, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 106775, + "latency_ms": 22.492, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 94928, + "latency_ms": 20.77, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 110221, + "latency_ms": 22.134, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87531, + "latency_ms": 20.609, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 102446, + "latency_ms": 20.279, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 83359, + "latency_ms": 20.333, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 66540, + "latency_ms": 19.888, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 101365, + "latency_ms": 21.054, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 88522, + "latency_ms": 20.553, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 92380, + "latency_ms": 19.759, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 82724, + "latency_ms": 28.734, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 88010, + "latency_ms": 29.888, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 84577, + "latency_ms": 24.241, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 88304, + "latency_ms": 22.286, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 88116, + "latency_ms": 22.276, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 98912, + "latency_ms": 22.557, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 86657, + "latency_ms": 22.021, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 90329, + "latency_ms": 21.367, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88014, + "latency_ms": 20.704, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 112651, + "latency_ms": 21.122, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "recovery_test.go" + ], + "missed": [ + "errors.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 115859, + "latency_ms": 21.896, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88387, + "latency_ms": 21.045, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 82791, + "latency_ms": 21.069, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 111258, + "latency_ms": 20.82, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88075, + "latency_ms": 21.262, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86659, + "latency_ms": 22.108, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 88259, + "latency_ms": 21.954, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 101364, + "latency_ms": 22.075, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 108196, + "latency_ms": 21.929, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 90059, + "latency_ms": 21.433, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [ + "recovery.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 72362, + "latency_ms": 20.837, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 105612, + "latency_ms": 21.381, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 106371, + "latency_ms": 21.866, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 92820, + "latency_ms": 20.742, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 83426, + "latency_ms": 19.983, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 84119, + "latency_ms": 20.868, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 113276, + "latency_ms": 22.396, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 113946, + "latency_ms": 20.821, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93998, + "latency_ms": 19.898, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 98912, + "latency_ms": 20.029, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 98262, + "latency_ms": 19.877, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "binding/protobuf.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 82704, + "latency_ms": 25.367, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 99455, + "latency_ms": 23.41, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 68234, + "latency_ms": 23.091, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 90299, + "latency_ms": 23.293, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go", + "logger_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 44600, + "latency_ms": 13.346, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12761, + "latency_ms": 13.518, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26285, + "latency_ms": 13.427, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24674, + "latency_ms": 14.525, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21162, + "latency_ms": 16.06, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 27850, + "latency_ms": 12.954, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28556, + "latency_ms": 13.153, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18035, + "latency_ms": 12.807, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19033, + "latency_ms": 12.346, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 37122, + "latency_ms": 12.269, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30316, + "latency_ms": 11.977, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 33394, + "latency_ms": 12.087, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26404, + "latency_ms": 12.219, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26292, + "latency_ms": 11.726, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22473, + "latency_ms": 11.684, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 15956, + "latency_ms": 11.877, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19388, + "latency_ms": 11.745, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17445, + "latency_ms": 12.487, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go" + ], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 27773, + "latency_ms": 11.689, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14497, + "latency_ms": 12.226, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30553, + "latency_ms": 11.962, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 37728, + "latency_ms": 10.761, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 34527, + "latency_ms": 10.572, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19547, + "latency_ms": 11.133, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12483, + "latency_ms": 11.281, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 22479, + "latency_ms": 10.652, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14254, + "latency_ms": 11.351, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 35496, + "latency_ms": 10.699, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [], + "missed": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23550, + "latency_ms": 11.323, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18818, + "latency_ms": 11.009, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 48958, + "latency_ms": 10.294, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53000, + "latency_ms": 11.107, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 28956, + "latency_ms": 10.251, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21545, + "latency_ms": 9.942, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 34999, + "latency_ms": 9.417, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 26477, + "latency_ms": 9.67, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20935, + "latency_ms": 10.459, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20327, + "latency_ms": 10.964, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21566, + "latency_ms": 10.685, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25538, + "latency_ms": 11.215, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24170, + "latency_ms": 10.813, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23086, + "latency_ms": 10.131, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20136, + "latency_ms": 11.979, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8753, + "latency_ms": 11.465, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35582, + "latency_ms": 10.816, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 27255, + "latency_ms": 11.326, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30122, + "latency_ms": 11.507, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [], + "missed": [ + "gin.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19879, + "latency_ms": 10.007, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33426, + "latency_ms": 11.389, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16642, + "latency_ms": 11.458, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27618, + "latency_ms": 10.152, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "errors.go", + "routergroup.go" + ], + "missed": [ + "context.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 39163, + "latency_ms": 10.865, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 16118, + "latency_ms": 9.775, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23279, + "latency_ms": 9.7, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 36634, + "latency_ms": 10.487, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 40307, + "latency_ms": 9.776, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23860, + "latency_ms": 9.307, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37903, + "latency_ms": 9.973, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49541, + "latency_ms": 11.629, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44167, + "latency_ms": 10.939, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13012, + "latency_ms": 9.794, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28129, + "latency_ms": 9.858, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19778, + "latency_ms": 10.244, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 27314, + "latency_ms": 10.268, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22773, + "latency_ms": 10.352, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 28023, + "latency_ms": 10.176, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24288, + "latency_ms": 9.838, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26606, + "latency_ms": 9.787, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21083, + "latency_ms": 9.957, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8705, + "latency_ms": 11.381, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55452, + "latency_ms": 10.964, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 25977, + "latency_ms": 10.091, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/protobuf.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 24180, + "latency_ms": 10.246, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16819, + "latency_ms": 10.253, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 15488, + "latency_ms": 10.207, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24911, + "latency_ms": 9.814, + "indexed_revision_matches": true + } + ], + "tier": "small", + "tier_code_files": 99, + "provenance": "Small-tier 12-arm run on gin. The codevetter-structural-context arm was driven through an experiment FIXTURE tool, not the shipped product surface; see caveat in candidates.json.", + "row_removed": "The codevetter row was measured through an experiment fixture AND under the adapter sort inversion; it is superseded by C-gin.json. Removed rather than kept, because averaging it with post-fix runs of the same provider yields a number describing neither. The other arms are unaffected and this file remains their only source." +} diff --git a/benchmarks/context-retrieval/results/small-gin-codevetter-real.json b/benchmarks/context-retrieval/results/small-gin-codevetter-real.json new file mode 100644 index 00000000..ce3418cd --- /dev/null +++ b/benchmarks/context-retrieval/results/small-gin-codevetter-real.json @@ -0,0 +1,6617 @@ +{ + "schema_version": "codevetter.context-retrieval-score.v1", + "repository": { + "id": "gin", + "head": "dcaa4296d111981ffb31ac3eba90bb63e1eb5ab9" + }, + "tier": "small", + "tier_code_files": 99, + "corpus_counts": { + "cases": 76, + "path_leak": 23, + "baseline_blind": 53, + "multi_file": 54 + }, + "cutoffs": [ + 5, + 10, + 20 + ], + "gates": { + "controls_present": { + "ok": true, + "missing": [], + "reason": null + }, + "controls_lose": { + "ok": true, + "best": 0.3914, + "control": 0.0263, + "reason": null + }, + "needs_raw_payload_check": [], + "nominated_for_hand_audit": [ + { + "provider_id": "random-files", + "query": "data race with trustedCIDRs (#2674) (#2675)", + "revision": "d496f64540b6707602de50ab57aeea8ff4080b74", + "returned": [ + "tree_test.go", + "binding/form_mapping_test.go", + "binding/form.go", + "routes_test.go", + "tree.go" + ] + }, + { + "provider_id": "random-files", + "query": "Fix #3500 Add escape logic for header (#3503)", + "revision": "fc1c43298de675e5252d0b44f97dc5e204bd4e1e", + "returned": [ + "mode_test.go", + "routes_test.go", + "internal/json/json.go", + "testdata/protoexample/test.pb.go", + "render/render.go" + ] + }, + { + "provider_id": "random-files", + "query": "check obj type in protobufBinding (#2851)", + "revision": "deb83b6365cf1b95f50929492f6058c94a6ff274", + "returned": [ + ".github/workflows/gin.yml", + "tree_test.go", + "render/xml.go", + "binding/protobuf.go", + "test_helpers.go" + ] + }, + { + "provider_id": "churn-ranked", + "query": "data race with trustedCIDRs (#2674) (#2675)", + "revision": "d496f64540b6707602de50ab57aeea8ff4080b74", + "returned": [ + "context.go", + "context_test.go", + "gin.go", + "binding/binding_test.go", + ".travis.yml" + ] + }, + { + "provider_id": "churn-ranked", + "query": "Do not panic when handling method not allowed on empty tree (#4003)", + "revision": "9c081de9cdd1948f521d47d170d18cbc2981c33a", + "returned": [ + "context.go", + "context_test.go", + "gin.go", + ".github/workflows/gin.yml", + "tree.go" + ] + }, + { + "provider_id": "churn-ranked", + "query": "check obj type in protobufBinding (#2851)", + "revision": "deb83b6365cf1b95f50929492f6058c94a6ff274", + "returned": [ + "context.go", + "context_test.go", + "gin.go", + "binding/binding_test.go", + "binding/form_mapping.go" + ] + } + ], + "trustworthy": true + }, + "providers": [ + { + "provider_id": "random-files", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.068, + "full_recall_at_5": 0.0132, + "mean_recall_at_10": 0.0921, + "full_recall_at_10": 0.0263, + "mean_recall_at_20": 0.1776, + "full_recall_at_20": 0.0526, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0263, + "mean_recall_at_16000_tokens": 0.1009, + "mean_precision_at_10": 0.0197, + "zero_hit_rate": 0.6842, + "median_tokens_delivered": 24792.5, + "median_latency_ms": 18.1355, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 74, + "mean_recall_at_5": 0.0563, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0676, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1554, + "full_recall_at_20": 0.027, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.027, + "mean_recall_at_16000_tokens": 0.0766, + "mean_precision_at_10": 0.0162, + "zero_hit_rate": 0.7027, + "median_tokens_delivered": 24792.5, + "median_latency_ms": 18.1355, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 1, + "full_recall_at_10": 1, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 1, + "mean_precision_at_10": 0.15, + "zero_hit_rate": 0, + "median_tokens_delivered": 21401, + "median_latency_ms": 18.384, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.0435, + "full_recall_at_5": 0, + "mean_recall_at_10": 0.0652, + "full_recall_at_10": 0, + "mean_recall_at_20": 0.1522, + "full_recall_at_20": 0.0435, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0217, + "mean_recall_at_16000_tokens": 0.0652, + "mean_precision_at_10": 0.013, + "zero_hit_rate": 0.7391, + "median_tokens_delivered": 24180, + "median_latency_ms": 18.335, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.0786, + "full_recall_at_5": 0.0189, + "mean_recall_at_10": 0.1038, + "full_recall_at_10": 0.0377, + "mean_recall_at_20": 0.1887, + "full_recall_at_20": 0.0566, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0283, + "mean_recall_at_16000_tokens": 0.1164, + "mean_precision_at_10": 0.0226, + "zero_hit_rate": 0.6604, + "median_tokens_delivered": 25977, + "median_latency_ms": 18.052, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 76 + }, + "resources": { + "baseline_rss_mb": 19404.7, + "peak_rss_mb": 19743.9, + "delta_rss_mb": 339.1, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "churn-ranked", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.3783, + "full_recall_at_5": 0.2632, + "mean_recall_at_10": 0.5077, + "full_recall_at_10": 0.3553, + "mean_recall_at_20": 0.7368, + "full_recall_at_20": 0.6579, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0175, + "mean_recall_at_16000_tokens": 0.2884, + "mean_precision_at_10": 0.0934, + "zero_hit_rate": 0.1842, + "median_tokens_delivered": 88454.5, + "median_latency_ms": 38.233, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 74, + "mean_recall_at_5": 0.375, + "full_recall_at_5": 0.2568, + "mean_recall_at_10": 0.5079, + "full_recall_at_10": 0.3514, + "mean_recall_at_20": 0.7297, + "full_recall_at_20": 0.6486, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.018, + "mean_recall_at_16000_tokens": 0.2962, + "mean_precision_at_10": 0.0946, + "zero_hit_rate": 0.1892, + "median_tokens_delivered": 88454.5, + "median_latency_ms": 38.1875, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 1, + "full_recall_at_20": 1, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0, + "mean_recall_at_16000_tokens": 0, + "mean_precision_at_10": 0.05, + "zero_hit_rate": 0, + "median_tokens_delivered": 87302.5, + "median_latency_ms": 42.625, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.4203, + "full_recall_at_5": 0.3478, + "mean_recall_at_10": 0.5507, + "full_recall_at_10": 0.3913, + "mean_recall_at_20": 0.7174, + "full_recall_at_20": 0.6087, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0435, + "mean_recall_at_16000_tokens": 0.3768, + "mean_precision_at_10": 0.0913, + "zero_hit_rate": 0.1739, + "median_tokens_delivered": 90059, + "median_latency_ms": 37.763, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.3601, + "full_recall_at_5": 0.2264, + "mean_recall_at_10": 0.489, + "full_recall_at_10": 0.3396, + "mean_recall_at_20": 0.7453, + "full_recall_at_20": 0.6792, + "mean_recall_at_1000_tokens": 0, + "mean_recall_at_4000_tokens": 0.0063, + "mean_recall_at_16000_tokens": 0.25, + "mean_precision_at_10": 0.0943, + "zero_hit_rate": 0.1887, + "median_tokens_delivered": 88304, + "median_latency_ms": 38.347, + "payload_kind": [ + "control-whole-files" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 76 + }, + "resources": { + "baseline_rss_mb": 19743.3, + "peak_rss_mb": 19858.3, + "delta_rss_mb": 115, + "scope": "system-wide-upper-bound", + "index_bytes_added": 0 + } + }, + { + "provider_id": "codevetter-structural-context", + "summary": { + "all": { + "cases": 76, + "mean_recall_at_5": 0.2818, + "full_recall_at_5": 0.1579, + "mean_recall_at_10": 0.3388, + "full_recall_at_10": 0.1974, + "mean_recall_at_20": 0.3958, + "full_recall_at_20": 0.25, + "mean_recall_at_1000_tokens": 0.3607, + "mean_recall_at_4000_tokens": 0.3914, + "mean_recall_at_16000_tokens": 0.3958, + "mean_precision_at_10": 0.1802, + "zero_hit_rate": 0.4605, + "median_tokens_delivered": 67.5, + "median_latency_ms": 350.8965, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_missed": { + "cases": 74, + "mean_recall_at_5": 0.2759, + "full_recall_at_5": 0.1486, + "mean_recall_at_10": 0.3345, + "full_recall_at_10": 0.1892, + "mean_recall_at_20": 0.393, + "full_recall_at_20": 0.2432, + "mean_recall_at_1000_tokens": 0.357, + "mean_recall_at_4000_tokens": 0.3885, + "mean_recall_at_16000_tokens": 0.393, + "mean_precision_at_10": 0.1817, + "zero_hit_rate": 0.4595, + "median_tokens_delivered": 69.5, + "median_latency_ms": 350.8965, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "baseline_solved": { + "cases": 2, + "mean_recall_at_5": 0.5, + "full_recall_at_5": 0.5, + "mean_recall_at_10": 0.5, + "full_recall_at_10": 0.5, + "mean_recall_at_20": 0.5, + "full_recall_at_20": 0.5, + "mean_recall_at_1000_tokens": 0.5, + "mean_recall_at_4000_tokens": 0.5, + "mean_recall_at_16000_tokens": 0.5, + "mean_precision_at_10": 0.125, + "zero_hit_rate": 0.5, + "median_tokens_delivered": 12, + "median_latency_ms": 305.366, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "path_leak": { + "cases": 23, + "mean_recall_at_5": 0.4094, + "full_recall_at_5": 0.3043, + "mean_recall_at_10": 0.4457, + "full_recall_at_10": 0.3043, + "mean_recall_at_20": 0.5906, + "full_recall_at_20": 0.4348, + "mean_recall_at_1000_tokens": 0.4746, + "mean_recall_at_4000_tokens": 0.5761, + "mean_recall_at_16000_tokens": 0.5906, + "mean_precision_at_10": 0.2507, + "zero_hit_rate": 0.2609, + "median_tokens_delivered": 99, + "median_latency_ms": 357.007, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + }, + "no_path_leak": { + "cases": 53, + "mean_recall_at_5": 0.2264, + "full_recall_at_5": 0.0943, + "mean_recall_at_10": 0.2925, + "full_recall_at_10": 0.1509, + "mean_recall_at_20": 0.3113, + "full_recall_at_20": 0.1698, + "mean_recall_at_1000_tokens": 0.3113, + "mean_recall_at_4000_tokens": 0.3113, + "mean_recall_at_16000_tokens": 0.3113, + "mean_precision_at_10": 0.1496, + "zero_hit_rate": 0.5472, + "median_tokens_delivered": 63, + "median_latency_ms": 348.778, + "payload_kind": [ + "excerpts" + ], + "unavailable": 0 + } + }, + "outcomes": { + "answered": 75, + "indexed-but-returned-nothing": 1 + }, + "resources": { + "baseline_rss_mb": 19859.8, + "peak_rss_mb": 20093.5, + "delta_rss_mb": 233.7, + "scope": "system-wide-upper-bound", + "index_bytes_added": 1087299584 + } + } + ], + "limitations": [ + "One repository; results describe this codebase, not codebases in general.", + "Ground truth is the files a real fix changed, which is a proxy for the files it had to find.", + "Commit subjects were written with the fix in hand, so queries are friendlier than real user prompts.", + "Retrieval quality is not task success; a provider can retrieve well and still not help an agent." + ], + "cases": [ + { + "provider_id": "random-files", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [ + "context.go", + "logger_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 10, + "tokens_delivered": 44600, + "latency_ms": 38.511, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12761, + "latency_ms": 17.781, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26285, + "latency_ms": 16.116, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24674, + "latency_ms": 18.363, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21162, + "latency_ms": 27.547, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 27850, + "latency_ms": 17.929, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28556, + "latency_ms": 19.017, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18035, + "latency_ms": 21.801, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19033, + "latency_ms": 18.256, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 37122, + "latency_ms": 17.621, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30316, + "latency_ms": 18.404, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 33394, + "latency_ms": 17.928, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26404, + "latency_ms": 18.317, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26292, + "latency_ms": 15.983, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22473, + "latency_ms": 16.622, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 15956, + "latency_ms": 17.383, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19388, + "latency_ms": 18.302, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 17445, + "latency_ms": 17.655, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go" + ], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 27773, + "latency_ms": 18.616, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14497, + "latency_ms": 16.451, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30553, + "latency_ms": 19.163, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 37728, + "latency_ms": 20.219, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 34527, + "latency_ms": 20.471, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19547, + "latency_ms": 18.633, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12483, + "latency_ms": 18.044, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 22479, + "latency_ms": 17.432, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 14254, + "latency_ms": 18.13, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.3333, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.1667, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 35496, + "latency_ms": 17.988, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [], + "missed": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23550, + "latency_ms": 17.116, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 18818, + "latency_ms": 17.19, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 48958, + "latency_ms": 18.458, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 53000, + "latency_ms": 17.614, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 28956, + "latency_ms": 17.378, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21545, + "latency_ms": 18.466, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 19, + "tokens_delivered": 34999, + "latency_ms": 18.335, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 26477, + "latency_ms": 16.82, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20935, + "latency_ms": 17.055, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20327, + "latency_ms": 18.141, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21566, + "latency_ms": 17.308, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25538, + "latency_ms": 16.891, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24170, + "latency_ms": 17.401, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23086, + "latency_ms": 17.564, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 20136, + "latency_ms": 16.482, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8753, + "latency_ms": 15.678, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 35582, + "latency_ms": 16.292, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 14, + "tokens_delivered": 27255, + "latency_ms": 18.052, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 30122, + "latency_ms": 16.389, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [], + "missed": [ + "gin.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19879, + "latency_ms": 17.56, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 33426, + "latency_ms": 17.444, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16642, + "latency_ms": 18.537, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 27618, + "latency_ms": 21.142, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "errors.go", + "routergroup.go" + ], + "missed": [ + "context.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 11, + "tokens_delivered": 39163, + "latency_ms": 20.291, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 16118, + "latency_ms": 19.623, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23279, + "latency_ms": 19.826, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 36634, + "latency_ms": 20.095, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 40307, + "latency_ms": 19.904, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 23860, + "latency_ms": 18.648, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 37903, + "latency_ms": 19.292, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 49541, + "latency_ms": 20.132, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 44167, + "latency_ms": 18.806, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13012, + "latency_ms": 17.852, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28129, + "latency_ms": 17.363, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 19778, + "latency_ms": 18.506, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 27314, + "latency_ms": 17.897, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 22773, + "latency_ms": 18.489, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 11, + "tokens_delivered": 28023, + "latency_ms": 17.745, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24288, + "latency_ms": 16.6, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 26606, + "latency_ms": 17.497, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 21083, + "latency_ms": 18.966, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8705, + "latency_ms": 20.377, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55452, + "latency_ms": 21.322, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 25977, + "latency_ms": 19.692, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/protobuf.go" + ], + "missed": [ + "binding/binding_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 24180, + "latency_ms": 18.586, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16819, + "latency_ms": 17.325, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 15488, + "latency_ms": 18.871, + "indexed_revision_matches": true + }, + { + "provider_id": "random-files", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 24911, + "latency_ms": 19.763, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "missed": [], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 1, + "tokens_delivered": 69810, + "latency_ms": 45.718, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 84576, + "latency_ms": 38.784, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87509, + "latency_ms": 38.289, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [ + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 82399, + "latency_ms": 36.049, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 83992, + "latency_ms": 38.04, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 72743, + "latency_ms": 38.661, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 88246, + "latency_ms": 36.717, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 105412, + "latency_ms": 37.615, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 84449, + "latency_ms": 38.216, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 95317, + "latency_ms": 44.017, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 69979, + "latency_ms": 42.885, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 113276, + "latency_ms": 39.956, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 108509, + "latency_ms": 37.371, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 84446, + "latency_ms": 39.324, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 90304, + "latency_ms": 38.614, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 89875, + "latency_ms": 38.159, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 70138, + "latency_ms": 39.613, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93950, + "latency_ms": 37.207, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 87314, + "latency_ms": 37.92, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 82768, + "latency_ms": 36.664, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 88101, + "latency_ms": 37.037, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go" + ], + "missed": [ + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 87276, + "latency_ms": 36.1, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 7, + "tokens_delivered": 117055, + "latency_ms": 36.529, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go", + "recovery_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 113658, + "latency_ms": 36.338, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 87849, + "latency_ms": 39.433, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 89097, + "latency_ms": 36.082, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93972, + "latency_ms": 35.892, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [ + "context_test.go", + "gin.go", + "routergroup.go" + ], + "missed": [ + "binding/default_validator.go", + "errors.go", + "response_writer.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.4, + "recall_at_10": 0.3333, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.15, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 88010, + "latency_ms": 35.853, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "gin.go" + ], + "missed": [ + "test_helpers.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 88138, + "latency_ms": 34.566, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [ + "debug.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 103341, + "latency_ms": 36.157, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 8, + "tokens_delivered": 115951, + "latency_ms": 37.763, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 106775, + "latency_ms": 40.83, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 94928, + "latency_ms": 37.862, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 15, + "tokens_delivered": 110221, + "latency_ms": 37.384, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 87531, + "latency_ms": 37.075, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 102446, + "latency_ms": 37.22, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 83359, + "latency_ms": 34.769, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 66540, + "latency_ms": 38.407, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 101365, + "latency_ms": 36.099, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 88522, + "latency_ms": 39.007, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "debug.go" + ], + "missed": [ + "mode.go", + "mode_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.3333, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 12, + "tokens_delivered": 92380, + "latency_ms": 36.706, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 82724, + "latency_ms": 36.684, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [ + "routergroup.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 88010, + "latency_ms": 36.636, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [ + "tree.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 84577, + "latency_ms": 36.772, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 88304, + "latency_ms": 37.401, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "gin.go", + "routes_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 88116, + "latency_ms": 36.059, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go", + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 98912, + "latency_ms": 38.25, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 86657, + "latency_ms": 36.698, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 90329, + "latency_ms": 37.168, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88014, + "latency_ms": 35.356, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 112651, + "latency_ms": 36.708, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [ + "context.go", + "recovery_test.go" + ], + "missed": [ + "errors.go", + "routergroup.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.2, + "recall_at_10": 0.25, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 1, + "tokens_delivered": 115859, + "latency_ms": 38.347, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88387, + "latency_ms": 37.826, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 82791, + "latency_ms": 37.214, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 111258, + "latency_ms": 43.542, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 88075, + "latency_ms": 44.871, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 86659, + "latency_ms": 46.256, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 20, + "tokens_delivered": 88259, + "latency_ms": 47.249, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 101364, + "latency_ms": 45.85, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 108196, + "latency_ms": 46.657, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 90059, + "latency_ms": 45.077, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [ + "recovery.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 16, + "tokens_delivered": 72362, + "latency_ms": 42.612, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 6, + "tokens_delivered": 105612, + "latency_ms": 43.146, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 106371, + "latency_ms": 42.025, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 92820, + "latency_ms": 40.206, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "githubapi_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 83426, + "latency_ms": 42.729, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 84119, + "latency_ms": 38.46, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 113276, + "latency_ms": 45.32, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 113946, + "latency_ms": 41.221, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 93998, + "latency_ms": 42.509, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [], + "missed": [ + "binding/toml.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 98912, + "latency_ms": 41.571, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 5, + "tokens_delivered": 98262, + "latency_ms": 42.475, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "binding/protobuf.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 4, + "tokens_delivered": 82704, + "latency_ms": 39.134, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 99455, + "latency_ms": 42.512, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [ + "debug.go", + "debug_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": 13, + "tokens_delivered": 68234, + "latency_ms": 43.225, + "indexed_revision_matches": true + }, + { + "provider_id": "churn-ranked", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [ + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 90299, + "latency_ms": 42.127, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-03e5e05ae089", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go", + "logger_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go", + "logger_test.go" + ], + "recall_at_5": 0.3333, + "precision_at_5": 0.2, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.3333, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.3333, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.3333, + "first_hit_rank": 2, + "tokens_delivered": 190, + "latency_ms": 275.645, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-088cdd74d42d", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 34, + "latency_ms": 339.218, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-09f8224593e3", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.3333, + "recall_at_20": 1, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 454, + "latency_ms": 359.271, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-11aa11a65618", + "path_leak": true, + "required_files": [ + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12, + "latency_ms": 328.695, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-1c2aa59b20c8", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [ + "routes_test.go", + "tree.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2222, + "recall_at_20": 1, + "precision_at_20": 0.2222, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 209, + "latency_ms": 339.498, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-1d0f938f28d7", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 63, + "latency_ms": 351.003, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-212267d67163", + "path_leak": false, + "required_files": [ + "context.go" + ], + "found": [], + "missed": [ + "context.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 329, + "latency_ms": 341.748, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-234a6d4c00cb", + "path_leak": true, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [ + "response_writer.go", + "response_writer_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 23, + "latency_ms": 343.559, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-24a1d2adb9db", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3, + "latency_ms": 348.778, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-28e57f58b184", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0714, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 6, + "tokens_delivered": 59, + "latency_ms": 321.1, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2921582d11d5", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.1818, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 8, + "tokens_delivered": 205, + "latency_ms": 268.28, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-293ad7edebb3", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 1, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 10, + "tokens_delivered": 758, + "latency_ms": 390.72, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2a794cd0b0fa", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28, + "latency_ms": 346.524, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2ae61570499d", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 762, + "latency_ms": 294.324, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2b1da2b0b38d", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1329, + "latency_ms": 299.054, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-2d4bbec94155", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [], + "missed": [ + "context.go", + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 28, + "latency_ms": 333.704, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-328d0b8076f0", + "path_leak": false, + "required_files": [ + "fs.go" + ], + "found": [], + "missed": [ + "fs.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8756, + "latency_ms": 319.625, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-36b0dede4b8c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 2827, + "latency_ms": 307.512, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-386d244068db", + "path_leak": false, + "required_files": [ + "routes_test.go", + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [ + "routes_test.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.6667, + "recall_at_10": 0.6667, + "precision_at_10": 0.6667, + "recall_at_20": 0.6667, + "precision_at_20": 0.6667, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 2, + "tokens_delivered": 13, + "latency_ms": 284.704, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-3a6f18f32f22", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 3758, + "latency_ms": 278.247, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-3dc1cd6572b4", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 126, + "latency_ms": 296.511, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-44d0dd70924d", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [], + "missed": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 54, + "latency_ms": 300.669, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-472d086af2ac", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go" + ], + "missed": [ + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0.5, + "precision_at_20": 0.0625, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 15, + "tokens_delivered": 308, + "latency_ms": 354.077, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-4a3eb31fb15b", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [], + "missed": [ + "recovery.go", + "recovery_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 10, + "latency_ms": 350.79, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-4a40f8f1a49b", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 0, + "latency_ms": 288.58, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-4cee78f5382d", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "routes_test.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 19, + "latency_ms": 313.243, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-4f339e6a35b1", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 30, + "latency_ms": 292.603, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-51aea73ba0f1", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "binding/default_validator.go", + "context_test.go", + "errors.go", + "gin.go", + "response_writer.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 60, + "latency_ms": 280.746, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-55e27f12465e", + "path_leak": true, + "required_files": [ + "context_test.go", + "gin.go", + "test_helpers.go" + ], + "found": [ + "context_test.go", + "test_helpers.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0.3333, + "precision_at_10": 0.1, + "recall_at_20": 0.6667, + "precision_at_20": 0.1, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0.3333, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 6, + "tokens_delivered": 5359, + "latency_ms": 340.806, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-5826722a87cf", + "path_leak": false, + "required_files": [ + "debug.go" + ], + "found": [], + "missed": [ + "debug.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 55, + "latency_ms": 325.076, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-5c00df8afadd", + "path_leak": true, + "required_files": [ + "render/data.go", + "render/render_test.go" + ], + "found": [ + "render/render_test.go" + ], + "missed": [ + "render/data.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 7, + "latency_ms": 355.657, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-5fad976b372e", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 67, + "latency_ms": 358.74, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-626d55b0c029", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [], + "missed": [ + "gin.go", + "gin_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 1086, + "latency_ms": 327.997, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-63dd3e60cab8", + "path_leak": false, + "required_files": [ + "recovery.go", + "recovery_test.go" + ], + "found": [ + "recovery.go" + ], + "missed": [ + "recovery_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 92, + "latency_ms": 362.815, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-646312aef6a3", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 18, + "tokens_delivered": 806, + "latency_ms": 357.007, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-674522db91d6", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.1053, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 17, + "tokens_delivered": 581, + "latency_ms": 408.523, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-68542126982e", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 2, + "latency_ms": 374.689, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7742ff50e0a0", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 1115, + "latency_ms": 340.588, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7a1b655074c3", + "path_leak": true, + "required_files": [ + "go.mod", + "go.sum", + "internal/json/json.go", + "internal/json/sonic.go" + ], + "found": [ + "internal/json/sonic.go" + ], + "missed": [ + "go.mod", + "go.sum", + "internal/json/json.go" + ], + "recall_at_5": 0.25, + "precision_at_5": 0.5, + "recall_at_10": 0.25, + "precision_at_10": 0.5, + "recall_at_20": 0.25, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.25, + "recall_at_4000_tokens": 0.25, + "recall_at_16000_tokens": 0.25, + "first_hit_rank": 2, + "tokens_delivered": 35, + "latency_ms": 428.111, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7cb151bb4c4c", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 329, + "latency_ms": 361.101, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7d147928ee23", + "path_leak": true, + "required_files": [ + "debug.go", + "mode.go", + "mode_test.go" + ], + "found": [ + "mode.go", + "mode_test.go" + ], + "missed": [ + "debug.go" + ], + "recall_at_5": 0.6667, + "precision_at_5": 0.4, + "recall_at_10": 0.6667, + "precision_at_10": 0.2, + "recall_at_20": 0.6667, + "precision_at_20": 0.1111, + "recall_at_1000_tokens": 0.6667, + "recall_at_4000_tokens": 0.6667, + "recall_at_16000_tokens": 0.6667, + "first_hit_rank": 4, + "tokens_delivered": 253, + "latency_ms": 360.105, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-7d189814cbf6", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.4, + "recall_at_20": 1, + "precision_at_20": 0.4, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 120, + "latency_ms": 344.857, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-814cd188eb0b", + "path_leak": false, + "required_files": [ + "routergroup.go" + ], + "found": [], + "missed": [ + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 25, + "latency_ms": 432.992, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-815122a0f4b9", + "path_leak": false, + "required_files": [ + "tree.go" + ], + "found": [], + "missed": [ + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 11, + "latency_ms": 362.687, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-82bcd6d39bfe", + "path_leak": false, + "required_files": [ + "binding/default_validator.go", + "binding/validate_test.go" + ], + "found": [ + "binding/validate_test.go" + ], + "missed": [ + "binding/default_validator.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 94, + "latency_ms": 377.33, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-86ff4a64c7ef", + "path_leak": false, + "required_files": [ + "gin.go", + "routes_test.go" + ], + "found": [ + "routes_test.go" + ], + "missed": [ + "gin.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.25, + "recall_at_10": 0.5, + "precision_at_10": 0.25, + "recall_at_20": 0.5, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 19, + "latency_ms": 364.099, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8763f33c65f7", + "path_leak": false, + "required_files": [ + "gin.go", + "gin_test.go" + ], + "found": [ + "gin.go" + ], + "missed": [ + "gin_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 2, + "tokens_delivered": 29, + "latency_ms": 428.926, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-87811a97bddb", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 1, + "precision_at_10": 0.125, + "recall_at_20": 1, + "precision_at_20": 0.125, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 7, + "tokens_delivered": 24, + "latency_ms": 360.421, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8790d08909fc", + "path_leak": true, + "required_files": [ + "binding/query.go", + "binding/query_test.go" + ], + "found": [], + "missed": [ + "binding/query.go", + "binding/query_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 45, + "latency_ms": 443.091, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8b9c55e8b059", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 903, + "latency_ms": 354.979, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8dd20118ba9a", + "path_leak": false, + "required_files": [ + "go.mod", + "go.sum" + ], + "found": [], + "missed": [ + "go.mod", + "go.sum" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 120, + "latency_ms": 446.623, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8e07d37c63e5", + "path_leak": false, + "required_files": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "found": [], + "missed": [ + "context.go", + "errors.go", + "recovery_test.go", + "routergroup.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 12193, + "latency_ms": 433.889, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-8eb5f832bac1", + "path_leak": true, + "required_files": [ + "routes_test.go", + "tree.go" + ], + "found": [], + "missed": [ + "routes_test.go", + "tree.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 99, + "latency_ms": 379.591, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-97b3c0d88ada", + "path_leak": true, + "required_files": [ + "context.go" + ], + "found": [ + "context.go" + ], + "missed": [], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 1, + "precision_at_20": 0.0556, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 11, + "tokens_delivered": 1700, + "latency_ms": 328.582, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-9914178584e4", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.2, + "recall_at_20": 0.5, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 23, + "latency_ms": 503.04, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-9f598a31aafb", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree_test.go" + ], + "missed": [ + "tree.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 52, + "latency_ms": 356.384, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-aa6002134e97", + "path_leak": false, + "required_files": [ + "context_go17_test.go" + ], + "found": [], + "missed": [ + "context_go17_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 78, + "latency_ms": 385.005, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-aefae309a4fc", + "path_leak": true, + "required_files": [ + "gin_test.go" + ], + "found": [ + "gin_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.2, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 4, + "tokens_delivered": 68, + "latency_ms": 365.848, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-b38c59de7fef", + "path_leak": false, + "required_files": [ + "errors.go", + "errors_test.go" + ], + "found": [], + "missed": [ + "errors.go", + "errors_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 994, + "latency_ms": 420.585, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-b917b14ff9d1", + "path_leak": false, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.0769, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 71, + "latency_ms": 436.319, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-bb1fc2e0fe97", + "path_leak": true, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go", + "context_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.4, + "recall_at_10": 1, + "precision_at_10": 0.2, + "recall_at_20": 1, + "precision_at_20": 0.2, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 2, + "tokens_delivered": 289, + "latency_ms": 397.017, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-be860ec1578f", + "path_leak": false, + "required_files": [ + "recovery.go" + ], + "found": [], + "missed": [ + "recovery.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 470, + "latency_ms": 338.385, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-c3d1092b3b48", + "path_leak": true, + "required_files": [ + "binding/form_mapping.go", + "binding/form_mapping_test.go" + ], + "found": [ + "binding/form_mapping_test.go" + ], + "missed": [ + "binding/form_mapping.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.3333, + "recall_at_10": 0.5, + "precision_at_10": 0.3333, + "recall_at_20": 0.5, + "precision_at_20": 0.3333, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 3, + "tokens_delivered": 5, + "latency_ms": 442.671, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-c3d5a28ed6d3", + "path_leak": false, + "required_files": [ + "gin.go" + ], + "found": [ + "gin.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.25, + "recall_at_10": 1, + "precision_at_10": 0.25, + "recall_at_20": 1, + "precision_at_20": 0.25, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 11, + "latency_ms": 439.599, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-c677ccc40a60", + "path_leak": false, + "required_files": [ + "go.mod" + ], + "found": [], + "missed": [ + "go.mod" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 16, + "latency_ms": 330.264, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-c706ace9298f", + "path_leak": false, + "required_files": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "found": [], + "missed": [ + "binding/binding_test.go", + "githubapi_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8, + "latency_ms": 277.915, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-cbdd47a7e108", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [ + "tree.go", + "tree_test.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 0.5, + "recall_at_10": 1, + "precision_at_10": 0.5, + "recall_at_20": 1, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 3, + "tokens_delivered": 56, + "latency_ms": 554.686, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-d75fcd4c9ab2", + "path_leak": false, + "required_files": [ + "response_writer.go", + "response_writer_test.go" + ], + "found": [], + "missed": [ + "response_writer.go", + "response_writer_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 280, + "latency_ms": 440.486, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-d9307dbcbbe7", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.5, + "recall_at_10": 0.5, + "precision_at_10": 0.5, + "recall_at_20": 0.5, + "precision_at_20": 0.5, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 5, + "latency_ms": 545.324, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-e0d46ded6cb6", + "path_leak": false, + "required_files": [ + "context.go", + "context_test.go" + ], + "found": [ + "context.go" + ], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 1, + "recall_at_10": 0.5, + "precision_at_10": 1, + "recall_at_20": 0.5, + "precision_at_20": 1, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 1, + "tokens_delivered": 4, + "latency_ms": 379.296, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-e737e3e267be", + "path_leak": true, + "required_files": [ + "binding/toml.go" + ], + "found": [ + "binding/toml.go" + ], + "missed": [], + "recall_at_5": 1, + "precision_at_5": 1, + "recall_at_10": 1, + "precision_at_10": 1, + "recall_at_20": 1, + "precision_at_20": 1, + "recall_at_1000_tokens": 1, + "recall_at_4000_tokens": 1, + "recall_at_16000_tokens": 1, + "first_hit_rank": 1, + "tokens_delivered": 9, + "latency_ms": 458.367, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-ea53388e6ee4", + "path_leak": false, + "required_files": [ + "tree.go", + "tree_test.go" + ], + "found": [], + "missed": [ + "tree.go", + "tree_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 856, + "latency_ms": 305.558, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-eab47b5423b6", + "path_leak": true, + "required_files": [ + "binding/binding_test.go", + "binding/protobuf.go" + ], + "found": [ + "binding/binding_test.go" + ], + "missed": [ + "binding/protobuf.go" + ], + "recall_at_5": 0.5, + "precision_at_5": 0.2, + "recall_at_10": 0.5, + "precision_at_10": 0.1, + "recall_at_20": 0.5, + "precision_at_20": 0.05, + "recall_at_1000_tokens": 0.5, + "recall_at_4000_tokens": 0.5, + "recall_at_16000_tokens": 0.5, + "first_hit_rank": 4, + "tokens_delivered": 509, + "latency_ms": 238.823, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-ebe5e2a6bfdc", + "path_leak": false, + "required_files": [ + ".golangci.yml" + ], + "found": [], + "missed": [ + ".golangci.yml" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 160, + "latency_ms": 344.262, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-f4bc259de33c", + "path_leak": false, + "required_files": [ + "debug.go", + "debug_test.go" + ], + "found": [], + "missed": [ + "debug.go", + "debug_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 13, + "latency_ms": 171.133, + "indexed_revision_matches": true + }, + { + "provider_id": "codevetter-structural-context", + "case_id": "gin-f70dd00b00bc", + "path_leak": true, + "required_files": [ + "context_test.go" + ], + "found": [], + "missed": [ + "context_test.go" + ], + "recall_at_5": 0, + "precision_at_5": 0, + "recall_at_10": 0, + "precision_at_10": 0, + "recall_at_20": 0, + "precision_at_20": 0, + "recall_at_1000_tokens": 0, + "recall_at_4000_tokens": 0, + "recall_at_16000_tokens": 0, + "first_hit_rank": null, + "tokens_delivered": 8, + "latency_ms": 182.761, + "indexed_revision_matches": true + } + ], + "superseded_by": "C-gin.json", + "superseded_reason": "Measured with the pre-improvement binary AND under the adapter sort inversion, so its codevetter row understates the arm twice over. Retained as the evidence behind the published retraction; excluded from aggregation via this marker." +}