From 092d4244d383197fd96056a1dac738db3af4cb60 Mon Sep 17 00:00:00 2001 From: Aneesh Date: Fri, 10 Jul 2026 13:52:11 -0400 Subject: [PATCH 1/2] fix: use correct storage path in run_benchmark.py --- benchmarks/run_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/run_benchmark.py b/benchmarks/run_benchmark.py index ad612b3..296da97 100644 --- a/benchmarks/run_benchmark.py +++ b/benchmarks/run_benchmark.py @@ -126,7 +126,7 @@ async def run_benchmark( print(f" Cache: {idx_result.cache_hits} hits, {idx_result.cache_misses} misses") # Set up retriever + compressor - storage_base = Path(config.storage_path) / project_dir.name + storage_base = project_storage_dir(config, project_dir) backend = LocalBackend(base_path=str(storage_base)) embedder = Embedder(model_name=config.embedding_model) retriever = HybridRetriever(backend=backend, embedder=embedder) From eda532e22715470482e3417284898a152ee84abd Mon Sep 17 00:00:00 2001 From: Aneesh Date: Fri, 10 Jul 2026 14:01:59 -0400 Subject: [PATCH 2/2] benchmark: add Django and Express benchmark results --- benchmarks/django_queries.json | 112 ++++++++++++++++++++++++++++++++ benchmarks/express_queries.json | 102 +++++++++++++++++++++++++++++ benchmarks/results/django.md | 76 ++++++++++++++++++++++ benchmarks/results/express.md | 74 +++++++++++++++++++++ 4 files changed, 364 insertions(+) create mode 100644 benchmarks/django_queries.json create mode 100644 benchmarks/express_queries.json create mode 100644 benchmarks/results/django.md create mode 100644 benchmarks/results/express.md diff --git a/benchmarks/django_queries.json b/benchmarks/django_queries.json new file mode 100644 index 0000000..825de58 --- /dev/null +++ b/benchmarks/django_queries.json @@ -0,0 +1,112 @@ +[ + { + "query": "How does Django's ORM build and compile SQL queries from a QuerySet?", + "expected_files": ["db/models/sql/compiler.py", "db/models/query.py"], + "category": "orm" + }, + { + "query": "How does Django's ModelBase metaclass and Model.save() assign and persist field values?", + "expected_files": ["db/models/base.py"], + "category": "orm" + }, + { + "query": "How does Django's MigrationExecutor.migrate() apply a migration plan?", + "expected_files": ["db/migrations/executor.py"], + "category": "migrations" + }, + { + "query": "How does Django's migration autodetector figure out model changes?", + "expected_files": ["db/migrations/autodetector.py"], + "category": "migrations" + }, + { + "query": "How does Django's URL resolver match a request path to a view?", + "expected_files": ["urls/resolvers.py"], + "category": "routing" + }, + { + "query": "How does Django convert URL path converters like into Python types?", + "expected_files": ["urls/converters.py"], + "category": "routing" + }, + { + "query": "How does Django's HttpRequest.build_absolute_uri() and get_host() determine the request URL?", + "expected_files": ["http/request.py"], + "category": "http" + }, + { + "query": "How does Django's HttpResponse class set headers and write content?", + "expected_files": ["http/response.py"], + "category": "http" + }, + { + "query": "How does Django parse multipart form data and file uploads?", + "expected_files": ["http/multipartparser.py"], + "category": "http" + }, + { + "query": "How does Django's authentication backend verify a user's credentials?", + "expected_files": ["contrib/auth/backends.py"], + "category": "auth" + }, + { + "query": "How does Django hash and validate user passwords?", + "expected_files": ["contrib/auth/hashers.py"], + "category": "auth" + }, + { + "query": "How does Django's session middleware load and persist session data?", + "expected_files": ["contrib/sessions/middleware.py", "contrib/sessions/base_session.py"], + "category": "sessions" + }, + { + "query": "How does Django's CSRF middleware protect against cross-site request forgery?", + "expected_files": ["middleware/csrf.py"], + "category": "middleware" + }, + { + "query": "How does Django's CommonMiddleware implement the APPEND_SLASH redirect?", + "expected_files": ["middleware/common.py"], + "category": "middleware" + }, + { + "query": "How does Django's Field.to_python() and validate() methods convert and check form input?", + "expected_files": ["forms/fields.py"], + "category": "forms" + }, + { + "query": "How does Django's ModelForm generate form fields from model fields?", + "expected_files": ["forms/models.py"], + "category": "forms" + }, + { + "query": "How does Django's template engine compile and render templates?", + "expected_files": ["template/engine.py", "template/base.py"], + "category": "templates" + }, + { + "query": "How does Django's ListView class paginate a queryset of objects?", + "expected_files": ["views/generic/list.py"], + "category": "views" + }, + { + "query": "How does Django's DetailView.get_object() and get_queryset() fetch a single model instance?", + "expected_files": ["views/generic/detail.py"], + "category": "views" + }, + { + "query": "How does Django's test client simulate HTTP requests for testing?", + "expected_files": ["test/client.py"], + "category": "testing" + }, + { + "query": "How does Django's signal dispatcher connect and send signals?", + "expected_files": ["dispatch/dispatcher.py"], + "category": "signals" + }, + { + "query": "How does Django's management command base class parse arguments and execute?", + "expected_files": ["core/management/base.py"], + "category": "management" + } +] diff --git a/benchmarks/express_queries.json b/benchmarks/express_queries.json new file mode 100644 index 0000000..78cb20d --- /dev/null +++ b/benchmarks/express_queries.json @@ -0,0 +1,102 @@ +[ + { + "query": "How does Express initialize the application and set up the default router?", + "expected_files": ["application.js"], + "category": "core" + }, + { + "query": "How does app.use() register middleware on an Express application?", + "expected_files": ["application.js"], + "category": "middleware" + }, + { + "query": "How does app.handle() dispatch an incoming request through the router?", + "expected_files": ["application.js"], + "category": "routing" + }, + { + "query": "How does app.set() and app.enabled() manage application settings?", + "expected_files": ["application.js"], + "category": "core" + }, + { + "query": "How does app.render() render a view with a template engine?", + "expected_files": ["application.js", "view.js"], + "category": "views" + }, + { + "query": "How does Express resolve and load view templates by name?", + "expected_files": ["view.js"], + "category": "views" + }, + { + "query": "How does res.send() determine the response body type and content-length?", + "expected_files": ["response.js"], + "category": "response" + }, + { + "query": "How does res.json() serialize an object and set the JSON content type?", + "expected_files": ["response.js"], + "category": "response" + }, + { + "query": "How does res.sendFile() stream a file to the client?", + "expected_files": ["response.js"], + "category": "response" + }, + { + "query": "How does res.download() set the Content-Disposition header for file downloads?", + "expected_files": ["response.js"], + "category": "response" + }, + { + "query": "How does res.status() and res.sendStatus() set the HTTP status code?", + "expected_files": ["response.js"], + "category": "response" + }, + { + "query": "How does res.cookie() and res.clearCookie() manage response cookies?", + "expected_files": ["response.js"], + "category": "response" + }, + { + "query": "How does req.accepts() perform content negotiation against the Accept header?", + "expected_files": ["request.js"], + "category": "request" + }, + { + "query": "How does req.get() and req.header() read request headers?", + "expected_files": ["request.js"], + "category": "request" + }, + { + "query": "How does req.is() check the request's Content-Type against a given type?", + "expected_files": ["request.js"], + "category": "request" + }, + { + "query": "How does req.range() parse a Range header for partial content requests?", + "expected_files": ["request.js"], + "category": "request" + }, + { + "query": "How does Express generate an ETag for a response body?", + "expected_files": ["utils.js"], + "category": "utils" + }, + { + "query": "How does Express compile and normalize configured MIME types?", + "expected_files": ["utils.js"], + "category": "utils" + }, + { + "query": "How does the express() factory function create an application instance?", + "expected_files": ["express.js"], + "category": "core" + }, + { + "query": "How does Express expose the json, urlencoded, and static middleware helpers?", + "expected_files": ["express.js"], + "category": "middleware" + } +] diff --git a/benchmarks/results/django.md b/benchmarks/results/django.md new file mode 100644 index 0000000..c798b79 --- /dev/null +++ b/benchmarks/results/django.md @@ -0,0 +1,76 @@ +# Benchmark: django + +**Date:** 2026-07-10 +**Project:** django (3655 files, 5,422,229 tokens) +**Index:** 13268 chunks from 2347 files in 117.9s + +## Results Summary + +| Metric | Value | +|--------|-------| +| Retrieval savings | **92.8%** (full files → relevant chunks) | +| Compression savings | **82.9%** (chunks → signatures) | +| Combined | **98.8%** (full files → compressed chunks) | +| Avg full-file baseline | 35,900 tokens/query | +| Avg after retrieval | 2,593 tokens/query | +| Avg after compression | 442 tokens/query | +| Precision@10 | 0.16 | +| Recall@10 | 0.95 | +| Latency p50 | 9.7ms | +| Queries tested | 22 | + +## Per-Layer Savings (each measured independently) + +Each layer has its own baseline. These are NOT stacked. + +| Layer | What it does | Savings | Method | +|-------|-------------|---------|--------| +| **Retrieval** | Full files → relevant code chunks | 93% | measured | +| **Chunk Compression** | Raw chunks → signatures + docstrings | 83% | measured | +| **Output Compression** | Reduces Claude's reply length | 70% | estimated | +| **Grammar** | Drops articles/fillers from memory text | 13% | measured | + +## Token Flow + +``` +Full files (avg): 35,900 tokens + → After retrieval: 2,593 tokens (92.8% saved) + → After compression: 442 tokens (82.9% more saved) +Combined savings: 98.8% +``` + +## Per-Query Results + +| Query | Full file | Chunks | Compressed | Retrieval | Compression | P@10 | R@10 | +|-------|-----------|--------|------------|-----------|-------------|------|------| +| How does Django's ORM build and compile | 58,057 | 2,773 | 368 | 95% | 87% | 0.12 | 0.50 | +| How does Django's ModelBase metaclass an | 68,264 | 2,696 | 426 | 96% | 84% | 0.14 | 1.00 | +| How does Django's MigrationExecutor.migr | 7,886 | 3,140 | 440 | 60% | 86% | 0.14 | 1.00 | +| How does Django's migration autodetector | 28,911 | 3,311 | 639 | 88% | 81% | 0.10 | 1.00 | +| How does Django's URL resolver match a r | 25,754 | 1,495 | 327 | 94% | 78% | 0.20 | 1.00 | +| How does Django convert URL path convert | 35,038 | 1,860 | 510 | 95% | 73% | 0.11 | 1.00 | +| How does Django's HttpRequest.build_abso | 57,672 | 2,271 | 446 | 96% | 80% | 0.11 | 1.00 | +| How does Django's HttpResponse class set | 29,515 | 2,585 | 339 | 91% | 87% | 0.14 | 1.00 | +| How does Django parse multipart form dat | 36,466 | 4,982 | 487 | 86% | 90% | 0.17 | 1.00 | +| How does Django's authentication backend | 25,201 | 904 | 319 | 96% | 65% | 0.20 | 1.00 | +| How does Django hash and validate user p | 19,501 | 1,412 | 455 | 93% | 68% | 0.17 | 1.00 | +| How does Django's session middleware loa | 11,208 | 2,250 | 362 | 80% | 84% | 0.12 | 0.50 | +| How does Django's CSRF middleware protec | 16,571 | 3,180 | 462 | 81% | 86% | 0.17 | 1.00 | +| How does Django's CommonMiddleware imple | 41,256 | 4,717 | 510 | 89% | 89% | 0.14 | 1.00 | +| How does Django's Field.to_python() and | 54,028 | 1,546 | 283 | 97% | 82% | 0.25 | 1.00 | +| How does Django's ModelForm generate for | 91,582 | 1,767 | 346 | 98% | 80% | 0.20 | 1.00 | +| How does Django's template engine compil | 26,696 | 882 | 380 | 97% | 57% | 0.29 | 1.00 | +| How does Django's ListView class paginat | 14,302 | 2,992 | 546 | 79% | 82% | 0.14 | 1.00 | +| How does Django's DetailView.get_object( | 57,495 | 1,418 | 550 | 98% | 61% | 0.17 | 1.00 | +| How does Django's test client simulate H | 45,902 | 4,420 | 489 | 90% | 89% | 0.20 | 1.00 | +| How does Django's signal dispatcher conn | 21,103 | 1,866 | 518 | 91% | 72% | 0.12 | 1.00 | +| How does Django's management command bas | 17,391 | 4,576 | 529 | 74% | 88% | 0.12 | 1.00 | + +## How to Reproduce + +```bash +pip install code-context-engine +python benchmarks/run_benchmark.py --repo https://github.com/django/django.git --source-dir django +``` + +Results generated by CCE benchmark suite on 2026-07-10. diff --git a/benchmarks/results/express.md b/benchmarks/results/express.md new file mode 100644 index 0000000..c075e72 --- /dev/null +++ b/benchmarks/results/express.md @@ -0,0 +1,74 @@ +# Benchmark: lib + +**Date:** 2026-07-10 +**Project:** lib (6 files, 15,521 tokens) +**Index:** 19 chunks from 6 files in 3.2s + +## Results Summary + +| Metric | Value | +|--------|-------| +| Retrieval savings | **93.6%** (full files → relevant chunks) | +| Compression savings | **54.3%** (chunks → signatures) | +| Combined | **97.1%** (full files → compressed chunks) | +| Avg full-file baseline | 14,908 tokens/query | +| Avg after retrieval | 961 tokens/query | +| Avg after compression | 439 tokens/query | +| Precision@10 | 0.18 | +| Recall@10 | 1.00 | +| Latency p50 | 2.7ms | +| Queries tested | 20 | + +## Per-Layer Savings (each measured independently) + +Each layer has its own baseline. These are NOT stacked. + +| Layer | What it does | Savings | Method | +|-------|-------------|---------|--------| +| **Retrieval** | Full files → relevant code chunks | 94% | measured | +| **Chunk Compression** | Raw chunks → signatures + docstrings | 54% | measured | +| **Output Compression** | Reduces Claude's reply length | 70% | estimated | +| **Grammar** | Drops articles/fillers from memory text | 13% | measured | + +## Token Flow + +``` +Full files (avg): 14,908 tokens + → After retrieval: 961 tokens (93.6% saved) + → After compression: 439 tokens (54.3% more saved) +Combined savings: 97.1% +``` + +## Per-Query Results + +| Query | Full file | Chunks | Compressed | Retrieval | Compression | P@10 | R@10 | +|-------|-----------|--------|------------|-----------|-------------|------|------| +| How does Express initialize the applicat | 15,521 | 813 | 435 | 95% | 46% | 0.17 | 1.00 | +| How does app.use() register middleware o | 15,521 | 706 | 426 | 96% | 40% | 0.17 | 1.00 | +| How does app.handle() dispatch an incomi | 15,521 | 1,050 | 433 | 93% | 59% | 0.17 | 1.00 | +| How does app.set() and app.enabled() man | 15,521 | 815 | 416 | 95% | 49% | 0.17 | 1.00 | +| How does app.render() render a view with | 15,521 | 679 | 399 | 96% | 41% | 0.33 | 1.00 | +| How does Express resolve and load view t | 15,521 | 851 | 440 | 94% | 48% | 0.17 | 1.00 | +| How does res.send() determine the respon | 12,458 | 1,396 | 550 | 89% | 61% | 0.20 | 1.00 | +| How does res.json() serialize an object | 15,521 | 1,310 | 476 | 92% | 64% | 0.17 | 1.00 | +| How does res.sendFile() stream a file to | 15,521 | 955 | 350 | 94% | 63% | 0.17 | 1.00 | +| How does res.download() set the Content- | 15,521 | 1,027 | 410 | 93% | 60% | 0.17 | 1.00 | +| How does res.status() and res.sendStatus | 12,458 | 981 | 472 | 92% | 52% | 0.20 | 1.00 | +| How does res.cookie() and res.clearCooki | 12,458 | 1,222 | 474 | 90% | 61% | 0.20 | 1.00 | +| How does req.accepts() perform content n | 15,521 | 1,183 | 447 | 92% | 62% | 0.17 | 1.00 | +| How does req.get() and req.header() read | 15,521 | 1,202 | 466 | 92% | 61% | 0.17 | 1.00 | +| How does req.is() check the request's Co | 15,521 | 967 | 470 | 94% | 51% | 0.17 | 1.00 | +| How does req.range() parse a Range heade | 15,521 | 1,183 | 447 | 92% | 62% | 0.17 | 1.00 | +| How does Express generate an ETag for a | 12,458 | 702 | 422 | 94% | 40% | 0.20 | 1.00 | +| How does Express compile and normalize c | 15,521 | 656 | 388 | 96% | 41% | 0.17 | 1.00 | +| How does the express() factory function | 15,521 | 718 | 438 | 95% | 39% | 0.17 | 1.00 | +| How does Express expose the json, urlenc | 15,521 | 807 | 429 | 95% | 47% | 0.17 | 1.00 | + +## How to Reproduce + +```bash +pip install code-context-engine +python benchmarks/run_benchmark.py --repo https://github.com/expressjs/express.git --source-dir lib +``` + +Results generated by CCE benchmark suite on 2026-07-10.